1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493 | package Torello.HTML.Tools.Images;
import Torello.Java.UnreachableError;
import Torello.Java.WritableDirectoryException;
import Torello.Java.FileRW;
import Torello.Java.Additional.AppendableLog;
import Torello.Java.Additional.AppendableSafe;
import java.net.URL;
import java.io.File;
// ************************************************************************************************
// ************************************************************************************************
// RECORD: Used as ImageScraper class Top-Level Data-Flow **AND** Helper-Function
// ************************************************************************************************
// ************************************************************************************************
// Simple "Record" that makes passing these parameters all around wily-nilly a lot easier
// Used Strictly Internally to this class
//
// There turns out to be a lot of "data" in both the form of "configurations", and even more
// that is saved and returned to the user after completion. This RECORD right here saves all
// of the data, and keeps inside ... well ... one single (top-level) reference.
class RECORD
{
private static final String I4 = " ";
// *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** ***
// Constant (final) for the ENTIRETY of the download-process
// *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** ***
final Request request;
final Results results;
final AppendableLog al;
final AppendableSafe log;
// Has a non-null log
final boolean hasLog;
// Verbosity-Level that is Strictly Equal-To
final boolean logLevelEQ1;
final boolean logLevelEQ2;
final boolean logLevelEQ3;
// Verbosity-Level that is Greater-Than or Equal-To
final boolean logLevelGTEQ1;
final boolean logLevelGTEQ2;
// *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** ***
// These change with each loop iteration
// *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** ***
// Reference-Fields
URL url = null;
String[] b64ImageData = null;
ImageInfo imageInfo = null;
// Boolean-Primitive
boolean isB64EncodedImage = false;
// *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** ***
// Constructor
// *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** ***
RECORD(Request request, Results results, AppendableLog al)
{
this.request = request;
this.results = results;
this.al = al;
this.log = al.log;
// If there is a non-null log, set the boolean stating that there is a log
this.hasLog = (al.log != null);
// DEBUGGING:
// System.out.println("hasLog: " + hasLog + ", al.level: " + al.level);
// if (! Q.YN("Continue?")) System.exit(0);
// Makes Verbose-Printing Code neater and easier to look at.
this.logLevelEQ1 = hasLog && (al.level == 1);
this.logLevelEQ2 = hasLog && (al.level == 2);
this.logLevelEQ3 = hasLog && (al.level == 3);
// Also Makes Verbosity Faster & Easier to Read
this.logLevelGTEQ1 = hasLog && (al.level >= 1);
this.logLevelGTEQ2 = hasLog && (al.level >= 2);
}
// *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** ***
// Some Simple Methods
// *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** ***
//
// This is called at the very beginning of the Primary Download-Loop, directly at the top
// of the loop-body. It s the first thing that is done on each iteration of the download.
//
// NOTE: This resets all NON-FINAL fields in this class.
void reset()
{
this.url = null;
this.b64ImageData = null;
this.imageInfo = null;
this.isB64EncodedImage = false;
}
void append(String s) { log.append(s); }
void appendI4(String s) { log.append(I4).append(s); }
// This is always a useful debugging tool, both now, and possibly in the future
public String toString()
{
return
// *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** ***
// Constant (final) for the ENTIRETY of the download-process
// *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** ***
"RECORD's 'final' Fields (Constant through-out entire-download):\n" +
// final Request request;
I4 + "this.request: " + ((this.request != null) ? "non-" : "") + "null\n" +
// final Results results;
I4 + "this.results: " + ((this.results != null) ? "non-" : "") + "null\n" +
// final AppendableLog al;
I4 + "this.AppendableLog: " + ((this.al != null) ? "non-" : "") + "null\n" +
// final AppendableSafe log;
I4 + "this.AppendableSafe: " + ((this.log != null) ? "non-" : "") + "null\n" +
// final boolean hasLog;
I4 + "this.hasLog: " + this.hasLog + '\n' +
// final boolean logLevelEQ1;
I4 + "this.logLevelEQ1: " + this.logLevelEQ1 + '\n' +
// final boolean logLevelEQ2;
I4 + "this.logLevelEQ2: " + this.logLevelEQ2 + '\n' +
// final boolean logLevelEQ3;
I4 + "this.logLevelEQ3: " + this.logLevelEQ3 + '\n' +
// final boolean logLevelGTEQ1;
I4 + "this.logLevelGTEQ1: " + this.logLevelGTEQ1 + '\n' +
// final boolean logLevelGTEQ2;
I4 + "this.logLevelGTEQ2: " + this.logLevelGTEQ2 + '\n' +
// *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** ***
// These change with each loop iteration
// *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** ***
"Fields that change on each Loop-Iteration:\n" +
// URL url = null;
I4 + "this.url: " +
((this.url != null) ? url.toString() : "null") + '\n' +
// String[] b64ImageData = null;
I4 + "this.b64ImageData: " +
((this.b64ImageData != null) ? "non-" : "") + "null\n" +
// ImageInfo imageInfo = null;
I4 + "this.imageInfo: " +
((this.imageInfo != null) ? "non-" : "") + "null\n" +
// boolean isB64Image = false;
I4 + "this.isB64EncodedImage: " + this.isB64EncodedImage + '\n';
}
// *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** ***
// Helpers that SIMULTANEOUSLY write-results to 'Results' and write-log to 'AppendableLog'
// *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** ***
// Simple Helper for Printing to the Appendable log
void printEx(String operation, Throwable t)
{
this.al.append(
" The " + operation + " Code has thrown an Exception:\n" +
" Throwable Class: " + t.getClass().getName() + '\n' +
" Message: [" + t.getMessage() + "]\n"
);
while ((t = t.getCause()) != null) this.al.append(
" Caused by Throwable Class: " + t.getClass().getName() + '\n' +
" Message: [" + t.getMessage() + "]\n"
);
}
// There are 4 different User-Provided Lambda-Targets. If they throw an exception (which
// should be extremely rare), this method is called.
//
// NOTE: This method only works if "RECORD.imageInfo" is NON-NULL. This means that the
// first couple of User-Provided Lambda's have to use "reportEx" instead!
<T> T userLambdaEx(String userLambdaName, Exception e) throws ImageScraperException
{
this.results.userLambdaException(this.imageInfo, e);
final String errMsg =
"While attempting to invoke the User-Provided Lambda-Target" +
"'Request." + userLambdaName + "', an exception was thrown by the code.";
if (this.request.skipOnUserLambdaException)
{
if (this.logLevelEQ1) this.append("x ");
else if (this.logLevelEQ2) this.appendI4(errMsg + '\n');
else if (this.logLevelEQ3) this.printEx("Invoke User '" + userLambdaName +"'", e);
return null;
}
else throw new ImageScraperException
(errMsg + ". Please see Throwable.getCause() for more details.", e);
}
<T> T reportEx(boolean skipBool, String errMsg, String operationName, Exception e)
throws ImageScraperException
{
// Paranoia & Sanity (A Simple Check) (An 'assert' that should never happen)
if (e == null) throw new UnreachableError();
this.results.exceptionFail(this.url, e);
if (skipBool)
{
if (this.logLevelEQ1) this.append("x ");
else if (this.logLevelEQ2) this.appendI4(errMsg + '\n');
else if (this.logLevelEQ3) this.printEx(operationName, e);
return null;
}
else throw ImageScraperException.class.isAssignableFrom(e.getClass())
? ((ImageScraperException) e)
: new ImageScraperException(errMsg + ". See Throwable.geCause() for details.", e);
}
// ********************************************************************************************
// ********************************************************************************************
// "Exception URL's" - Rare, but happens if the Static-Builder threw an Exception
// ********************************************************************************************
// ********************************************************************************************
void dealWithExceptionURL()
{
// "Exception-URL's" are URL's that must have come from the static "TagNode"
// Builders in class Request. It happens when a complete-URL cannot be built
// from a partial-URL, and the Links-Class saved the Exception in a Vector,
// so that it can be reported to the user (righ here!)
Exception e = this.request.nextTNSRCException();
// ASSERT-STATEMENT: The 'request' instance should always return an 'e' here
if (e == null) throw new UnreachableError();
// Since this "Failed", make sure to let the "Results" object-instance know.
this.results.tagNodeSRCError(e);
// Now let the user know too
if (this.hasLog)
{
if (this.logLevelEQ1) this.append(" x ");
else if (this.logLevelGTEQ2) this.append
("URL-Building Exception: " + e.getClass().getName() + '\n');
if (this.logLevelEQ3) this.appendI4("Message: " + e.getMessage() + '\n');
}
}
// ********************************************************************************************
// ********************************************************************************************
// User-Provided URL-PreProcessor (Maybe!)
// ********************************************************************************************
// ********************************************************************************************
void doUserURLPreProcessorIfNeeded() throws ImageScraperException
{
if ((this.url == null) || (this.request.urlPreProcessor == null)) return;
try
{
this.url = this.request.urlPreProcessor.apply(this.url);
if (this.logLevelGTEQ2) this.appendI4("Pre-Processor URL:" + this.url + '\n');
}
catch (Exception e)
{
final String msg =
"While attempting to invoke the user provided lambda " +
"'Request.urlPreProcessor', an exception was thrown by the user-code.";
if (this.request.skipOnUserLambdaException)
{
if (this.logLevelGTEQ2) this.appendI4(msg);
if (this.logLevelEQ3) this.printEx("Run URL-PreProcessor", e);
if (this.logLevelEQ1) this.append("x ");
this.results.exceptionFail(this.url, e);
return;
}
else throw new ImageScraperException
(msg + " Please see Throwable.getCause() for more details.", e);
}
}
// ********************************************************************************************
// ********************************************************************************************
// Write to Disk, or Send to Request.imageReceiver
// ********************************************************************************************
// ********************************************************************************************
void handleImageByteArray() throws ImageScraperException
{
// *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** ***
// Get the File-Name, this likely is an "error-free" step, but check just in case.
// *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** ***
final String tempFileName = ComputeFileName.run(
this.request,
this.imageInfo,
this.results.successCounter,
(Exception e) -> this.userLambdaEx("getImageFileSaveName", e)
);
if (tempFileName == null) return;
this.imageInfo.setFileName(tempFileName);
// *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** ***
// Run the User's Keeper-Predicate, if one was supplied
// *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** ***
boolean keepIt = true;
if (this.request.keeperPredicate != null)
try
{ keepIt = this.request.keeperPredicate.test(this.imageInfo); }
catch (Exception e)
{ this.userLambdaEx("keeperPredicate", e); return;}
// *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** ***
// Write-Image, or send to Request.imageReceiver
// *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** ***
if (! keepIt)
{
this.results.predicateReject(this.imageInfo);
// Now let the user-log know... (MAYBE, IF THEY HAVE LEVEL-CLEARANCE)
if (this.logLevelEQ3)
this.appendI4("User-Provided Keeper Predicate Rejected this Image.");
if (this.logLevelEQ1)
this.append("x ");
}
else this.writeOrTransmit();
}
private void writeOrTransmit() throws ImageScraperException
{
// *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** ***
// Case: ImageReceiver
// *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** ***
if (this.request.imageReceiver != null)
try
{
this.request.imageReceiver.accept(this.imageInfo);
this.results.success(this.imageInfo, null /* no target directory */);
if (this.logLevelEQ1) this.append("✓ ");
else if (this.logLevelGTEQ2)
this.appendI4("Image Properly Transmitted to Request.imageReceiver\n");
return;
}
catch (Exception e)
{ this.userLambdaEx("imageReceiver", e); return; }
// *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** ***
// Case: File-System
// *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** ***
String dirName = null;
if (this.request.targetDirectory != null) dirName = this.request.targetDirectory;
else if (this.request.targetDirectoryRetriever != null)
{
final File dir;
// *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** ***
// Run the Request.targetDirectoryRetriever instance
// *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** ***
try
{ dir = this.request.targetDirectoryRetriever.apply(this.imageInfo); }
catch (Exception e)
{ this.userLambdaEx("targetDirectoryRetriever", e); return; }
// *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** ***
// Check that the directory returned is non-null and writeable
// *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** ***
try
{ WritableDirectoryException.check(dir); }
catch (Exception e)
{
// This call either returns null, or throws an ImageScraperException
// Depending upon the boolean 'this.request.skipOnImageWritingFail'
this.reportEx(
this.request.skipOnImageWritingFail,
"Target-Directory reference provided is not a File-System Writeable Directory",
"Write Image to Disk",
e
);
}
}
// This scenario is checked inside the Request class "check" method
else throw new UnreachableError();
// *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** ***
// WRITE THE FILE
// *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** ***
if (! dirName.endsWith(File.separator)) dirName = dirName + File.separator;
try
{
final String saveName =
dirName +
this.imageInfo.fileName() + '.' +
this.imageInfo.actualExtension.extension;
FileRW.writeBinary(this.imageInfo.imgByteArr, saveName);
this.results.success(this.imageInfo, dirName);
if (this.logLevelEQ1) this.append("✓ ");
else if (this.logLevelGTEQ2)
this.appendI4("Image saved successfully to: [" + saveName + "]\n");
}
catch (Exception e)
{
// This call either returns null, or throws an ImageScraperException
// Depending upon the boolean 'this.request.skipOnImageWritingFail'
this.reportEx(
this.request.skipOnImageWritingFail,
"Exception thrown while attempting to write an image file to disk.",
"Write Image to Disk",
e
);
}
}
}
|