Debian Codegraph @

src/warc.c

raw
1 /* Utility functions for writing WARC files.
2 Copyright (C) 2011-2012, 2015, 2018-2024 Free Software Foundation,
3 Inc.
4
5 This file is part of GNU Wget.
6
7 GNU Wget is free software; you can redistribute it and/or modify
8 it under the terms of the GNU General Public License as published by
9 the Free Software Foundation; either version 3 of the License, or (at
10 your option) any later version.
11
12 GNU Wget is distributed in the hope that it will be useful,
13 but WITHOUT ANY WARRANTY; without even the implied warranty of
14 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15 GNU General Public License for more details.
16
17 You should have received a copy of the GNU General Public License
18 along with Wget. If not, see <http://www.gnu.org/licenses/>.
19
20 Additional permission under GNU GPL version 3 section 7
21
22 If you modify this program, or any covered work, by linking or
23 combining it with the OpenSSL project's OpenSSL library (or a
24 modified version of that library), containing parts covered by the
25 terms of the OpenSSL or SSLeay licenses, the Free Software Foundation
26 grants you additional permission to convey the resulting work.
27 Corresponding Source for a non-source form of such a combination
28 shall include the source code for the parts of OpenSSL used as well
29 as that of the covered work. */
30
31 #include "wget.h"
32 #include "hash.h"
33 #include "utils.h"
34 #include "version.h"
35 #include "dirname.h"
36 #include "url.h"
37
38 #include <stdio.h>
39 #include <stdlib.h>
40 #include <string.h>
41 #include <time.h>
42 #include <tmpdir.h>
43 #include <sha1.h>
44 #include <base32.h>
45 #include <unistd.h>
46 #ifdef HAVE_LIBZ
47 #include <zlib.h>
48 #endif
49
50 #ifdef HAVE_LIBUUID
51 #include <uuid/uuid.h>
52 #elif HAVE_UUID_CREATE
53 #include <uuid.h>
54 #endif
55
56 #include "warc.h"
57 #include "exits.h"
58
59 #ifdef WINDOWS
60 /* we need this on Windows to have O_TEMPORARY defined */
61 # include <fcntl.h>
62 # include <rpc.h>
63 #endif
64
65 #ifndef O_TEMPORARY
66 #define O_TEMPORARY 0
67 #endif
68
69 #include "warc.h"
70 #include "exits.h"
71
72
73 /* The log file (a temporary file that contains a copy
74 of the wget log). */
75 static FILE *warc_log_fp;
76
77 /* The manifest file (a temporary file that contains the
78 warcinfo uuid of every file in this crawl). */
79 static FILE *warc_manifest_fp;
80
81 /* The current WARC file (or NULL, if WARC is disabled). */
82 static FILE *warc_current_file;
83
84 #ifdef HAVE_LIBZ
85 /* The gzip stream for the current WARC file
86 (or NULL, if WARC or gzip is disabled). */
87 static gzFile warc_current_gzfile;
88
89 /* The offset of the current gzip record in the WARC file. */
90 static off_t warc_current_gzfile_offset;
91
92 /* The uncompressed size (so far) of the current record. */
93 static off_t warc_current_gzfile_uncompressed_size;
94 # endif
95
96 /* This is true until a warc_write_* method fails. */
97 static bool warc_write_ok;
98
99 /* The current CDX file (or NULL, if CDX is disabled). */
100 static FILE *warc_current_cdx_file;
101
102 /* The record id of the warcinfo record of the current WARC file. */
103 static char warc_current_warcinfo_uuid_str[48];
104
105 /* The file name of the current WARC file. */
106 static char *warc_current_filename;
107
108 /* The serial number of the current WARC file. This number is
109 incremented each time a new file is opened and is used in the
110 WARC file's filename. */
111 static int warc_current_file_number;
112
113 /* The table of CDX records, if deduplication is enabled. */
114 static struct hash_table * warc_cdx_dedup_table;
115
116 static bool warc_start_new_file (bool meta);
117
118
119 struct warc_cdx_record
120 {
121 char *url;
122 char *uuid;
123 char digest[SHA1_DIGEST_SIZE];
124 };
125
126 static unsigned long
127 warc_hash_sha1_digest (const void *key)
128 {
129 /* We just use some of the first bytes of the digest. */
130 unsigned long v = 0;
131 memcpy (&v, key, sizeof (unsigned long));
132 return v;
133 }
134
135 static int
136 warc_cmp_sha1_digest (const void *digest1, const void *digest2)
137 {
138 return !memcmp (digest1, digest2, SHA1_DIGEST_SIZE);
139 }
140
141
142
143 /* Writes SIZE bytes from BUFFER to the current WARC file,
144 through gzwrite if compression is enabled.
145 Returns the number of uncompressed bytes written. */
146 static size_t
147 warc_write_buffer (const char *buffer, size_t size)
148 {
149 #ifdef HAVE_LIBZ
150 if (warc_current_gzfile)
151 {
152 warc_current_gzfile_uncompressed_size += size;
153 return gzwrite (warc_current_gzfile, buffer, size);
154 }
155 else
156 #endif
157 return fwrite (buffer, 1, size, warc_current_file);
158 }
159
160 /* Writes STR to the current WARC file.
161 Returns false and set warc_write_ok to false if there
162 is an error. */
163 static bool
164 warc_write_string (const char *str)
165 {
166 size_t n;
167
168 if (!warc_write_ok)
169 return false;
170
171 n = strlen (str);
172 if (n != warc_write_buffer (str, n))
173 warc_write_ok = false;
174
175 return warc_write_ok;
176 }
177
178
179 #define EXTRA_GZIP_HEADER_SIZE 14
180 #define GZIP_STATIC_HEADER_SIZE 10
181 #define FLG_FEXTRA 0x04
182 #define OFF_FLG 3
183
184 /* Starts a new WARC record. Writes the version header.
185 If opt.warc_maxsize is set and the current file is becoming
186 too large, this will open a new WARC file.
187
188 If compression is enabled, this will start a new
189 gzip stream in the current WARC file.
190
191 Returns false and set warc_write_ok to false if there
192 is an error. */
193 static bool
194 warc_write_start_record (void)
195 {
196 if (!warc_write_ok)
197 return false;
198
199 fflush (warc_current_file);
200 if (opt.warc_maxsize > 0 && ftello (warc_current_file) >= opt.warc_maxsize)
201 warc_start_new_file (false);
202
203 #ifdef HAVE_LIBZ
204 /* Start a GZIP stream, if required. */
205 if (opt.warc_compression_enabled)
206 {
207 int dup_fd;
208 /* Record the starting offset of the new record. */
209 warc_current_gzfile_offset = ftello (warc_current_file);
210
211 /* Reserve space for the extra GZIP header field.
212 In warc_write_end_record we will fill this space
213 with information about the uncompressed and
214 compressed size of the record. */
215 if (fseek (warc_current_file, EXTRA_GZIP_HEADER_SIZE, SEEK_CUR) < 0)
216 {
217 logprintf (LOG_NOTQUIET, _("Error setting WARC file position.\n"));
218 warc_write_ok = false;
219 return false;
220 }
221
222 if (fflush (warc_current_file) != 0)
223 {
224 logprintf (LOG_NOTQUIET, _("Error flushing WARC file to disk.\n"));
225 warc_write_ok = false;
226 return false;
227 }
228
229 /* Start a new GZIP stream. */
230 dup_fd = dup (fileno (warc_current_file));
231 if (dup_fd < 0)
232 {
233 logprintf (LOG_NOTQUIET,
234 _("Error duplicating WARC file file descriptor.\n"));
235 warc_write_ok = false;
236 return false;
237 }
238
239 warc_current_gzfile = gzdopen (dup_fd, "wb9");
240 warc_current_gzfile_uncompressed_size = 0;
241
242 if (warc_current_gzfile == NULL)
243 {
244 logprintf (LOG_NOTQUIET,
245 _("Error opening GZIP stream to WARC file.\n"));
246 close (dup_fd);
247 warc_write_ok = false;
248 return false;
249 }
250 }
251 #endif
252
253 warc_write_string ("WARC/1.0\r\n");
254 return warc_write_ok;
255 }
256
257 /* Writes a WARC header to the current WARC record.
258 This method may be run after warc_write_start_record and
259 before warc_write_block_from_file. */
260 static bool
261 warc_write_header (const char *name, const char *value)
262 {
263 if (value)
264 {
265 warc_write_string (name);
266 warc_write_string (": ");
267 warc_write_string (value);
268 warc_write_string ("\r\n");
269 }
270 return warc_write_ok;
271 }
272
273 /* Writes a WARC header with a URI as value to the current WARC record.
274 This method may be run after warc_write_start_record and
275 before warc_write_block_from_file. */
276 static bool
277 warc_write_header_uri (const char *name, const char *value)
278 {
279 if (value)
280 {
281 warc_write_string (name);
282 warc_write_string (": <");
283 warc_write_string (value);
284 warc_write_string (">\r\n");
285 }
286 return warc_write_ok;
287 }
288
289 /* Copies the contents of DATA_IN to the WARC record.
290 Adds a Content-Length header to the WARC record.
291 Run this method after warc_write_header,
292 then run warc_write_end_record. */
293 static bool
294 warc_write_block_from_file (FILE *data_in)
295 {
296 /* Add the Content-Length header. */
297 char content_length[MAX_INT_TO_STRING_LEN(off_t)];
298 char buffer[BUFSIZ];
299 size_t s;
300
301 fseeko (data_in, 0L, SEEK_END);
302 number_to_string (content_length, ftello (data_in));
303 warc_write_header ("Content-Length", content_length);
304
305 /* End of the WARC header section. */
306 warc_write_string ("\r\n");
307
308 if (fseeko (data_in, 0L, SEEK_SET) != 0)
309 warc_write_ok = false;
310
311 /* Copy the data in the file to the WARC record. */
312 while (warc_write_ok && (s = fread (buffer, 1, BUFSIZ, data_in)) > 0)
313 {
314 if (warc_write_buffer (buffer, s) < s)
315 warc_write_ok = false;
316 }
317
318 return warc_write_ok;
319 }
320
321 /* Run this method to close the current WARC record.
322
323 If compression is enabled, this method closes the
324 current GZIP stream and fills the extra GZIP header
325 with the uncompressed and compressed length of the
326 record. */
327 static bool
328 warc_write_end_record (void)
329 {
330 if (!warc_write_ok)
331 return warc_write_ok;
332
333 if (warc_write_buffer ("\r\n\r\n", 4) != 4)
334 {
335 warc_write_ok = false;
336 return false;
337 }
338
339 #ifdef HAVE_LIBZ
340 /* We start a new gzip stream for each record. */
341 if (warc_write_ok && warc_current_gzfile)
342 {
343 char extra_header[EXTRA_GZIP_HEADER_SIZE];
344 char static_header[GZIP_STATIC_HEADER_SIZE];
345 off_t current_offset, uncompressed_size, compressed_size;
346 size_t result;
347
348 if (gzclose (warc_current_gzfile) != Z_OK)
349 {
350 warc_write_ok = false;
351 return false;
352 }
353
354 fflush (warc_current_file);
355 fseeko (warc_current_file, 0, SEEK_END);
356
357 /* The WARC standard suggests that we add 'skip length' data in the
358 extra header field of the GZIP stream.
359
360 In warc_write_start_record we reserved space for this extra header.
361 This extra space starts at warc_current_gzfile_offset and fills
362 EXTRA_GZIP_HEADER_SIZE bytes. The static GZIP header starts at
363 warc_current_gzfile_offset + EXTRA_GZIP_HEADER_SIZE.
364
365 We need to do three things:
366 1. Move the static GZIP header to warc_current_gzfile_offset;
367 2. Set the FEXTRA flag in the GZIP header;
368 3. Write the extra GZIP header after the static header, that is,
369 starting at warc_current_gzfile_offset + GZIP_STATIC_HEADER_SIZE.
370 */
371
372 /* Calculate the uncompressed and compressed sizes. */
373 current_offset = ftello (warc_current_file);
374 uncompressed_size = current_offset - warc_current_gzfile_offset;
375 compressed_size = warc_current_gzfile_uncompressed_size;
376
377 /* Go back to the static GZIP header. */
378 result = fseeko (warc_current_file, warc_current_gzfile_offset
379 + EXTRA_GZIP_HEADER_SIZE, SEEK_SET);
380 if (result != 0)
381 {
382 warc_write_ok = false;
383 return false;
384 }
385
386 /* Read the header. */
387 result = fread (static_header, 1, GZIP_STATIC_HEADER_SIZE,
388 warc_current_file);
389 if (result != GZIP_STATIC_HEADER_SIZE)
390 {
391 warc_write_ok = false;
392 return false;
393 }
394
395 /* Set the FEXTRA flag in the flags byte of the header. */
396 static_header[OFF_FLG] = static_header[OFF_FLG] | FLG_FEXTRA;
397
398 /* Write the header back to the file, but starting at
399 warc_current_gzfile_offset. */
400 fseeko (warc_current_file, warc_current_gzfile_offset, SEEK_SET);
401 fwrite (static_header, 1, GZIP_STATIC_HEADER_SIZE, warc_current_file);
402
403 /* Prepare the extra GZIP header. */
404 /* XLEN, the length of the extra header fields. */
405 extra_header[0] = ((EXTRA_GZIP_HEADER_SIZE - 2) & 255);
406 extra_header[1] = ((EXTRA_GZIP_HEADER_SIZE - 2) >> 8) & 255;
407 /* The extra header field identifier for the WARC skip length. */
408 extra_header[2] = 's';
409 extra_header[3] = 'l';
410 /* The size of the field value (8 bytes). */
411 extra_header[4] = (8 & 255);
412 extra_header[5] = ((8 >> 8) & 255);
413 /* The size of the uncompressed record. */
414 extra_header[6] = (uncompressed_size & 255);
415 extra_header[7] = (uncompressed_size >> 8) & 255;
416 extra_header[8] = (uncompressed_size >> 16) & 255;
417 extra_header[9] = (uncompressed_size >> 24) & 255;
418 /* The size of the compressed record. */
419 extra_header[10] = (compressed_size & 255);
420 extra_header[11] = (compressed_size >> 8) & 255;
421 extra_header[12] = (compressed_size >> 16) & 255;
422 extra_header[13] = (compressed_size >> 24) & 255;
423
424 /* Write the extra header after the static header. */
425 fseeko (warc_current_file, warc_current_gzfile_offset
426 + GZIP_STATIC_HEADER_SIZE, SEEK_SET);
427 fwrite (extra_header, 1, EXTRA_GZIP_HEADER_SIZE, warc_current_file);
428
429 /* Done, move back to the end of the file. */
430 fflush (warc_current_file);
431 fseeko (warc_current_file, 0, SEEK_END);
432 }
433 #endif /* HAVE_LIBZ */
434
435 return warc_write_ok;
436 }
437
438
439 /* Writes the WARC-Date header for the given timestamp to
440 the current WARC record.
441 If timestamp is NULL, the current time will be used. */
442 static bool
443 warc_write_date_header (const char *timestamp)
444 {
445 char current_timestamp[21];
446
447 return warc_write_header ("WARC-Date", timestamp ? timestamp :
448 warc_timestamp (current_timestamp, sizeof(current_timestamp)));
449 }
450
451 /* Writes the WARC-IP-Address header for the given IP to
452 the current WARC record. If IP is NULL, no header will
453 be written. */
454 static bool
455 warc_write_ip_header (const ip_address *ip)
456 {
457 if (ip != NULL)
458 return warc_write_header ("WARC-IP-Address", print_address (ip));
459 else
460 return warc_write_ok;
461 }
462
463
464 /* warc_sha1_stream_with_payload is a modified copy of sha1_stream
465 from gnulib/sha1.c. This version calculates two digests in one go.
466
467 Compute SHA1 message digests for bytes read from STREAM. The
468 digest of the complete file will be written into the 16 bytes
469 beginning at RES_BLOCK.
470
471 If payload_offset >= 0, a second digest will be calculated of the
472 portion of the file starting at payload_offset and continuing to
473 the end of the file. The digest number will be written into the
474 16 bytes beginning ad RES_PAYLOAD. */
475 static int
476 warc_sha1_stream_with_payload (FILE *stream, void *res_block, void *res_payload,
477 off_t payload_offset)
478 {
479 #define BLOCKSIZE 32768
480
481 struct sha1_ctx ctx_block;
482 struct sha1_ctx ctx_payload;
483 off_t pos;
484 off_t sum;
485
486 char *buffer = xmalloc (BLOCKSIZE + 72);
487
488 /* Initialize the computation context. */
489 sha1_init_ctx (&ctx_block);
490 if (payload_offset >= 0)
491 sha1_init_ctx (&ctx_payload);
492
493 pos = 0;
494
495 /* Iterate over full file contents. */
496 while (1)
497 {
498 /* We read the file in blocks of BLOCKSIZE bytes. One call of the
499 computation function processes the whole buffer so that with the
500 next round of the loop another block can be read. */
501 off_t n;
502 sum = 0;
503
504 /* Read block. Take care for partial reads. */
505 while (1)
506 {
507 n = fread (buffer + sum, 1, BLOCKSIZE - sum, stream);
508
509 sum += n;
510 pos += n;
511
512 if (sum == BLOCKSIZE)
513 break;
514
515 if (n == 0)
516 {
517 /* Check for the error flag IF N == 0, so that we don't
518 exit the loop after a partial read due to e.g., EAGAIN
519 or EWOULDBLOCK. */
520 if (ferror (stream))
521 {
522 xfree (buffer);
523 return 1;
524 }
525 goto process_partial_block;
526 }
527
528 /* We've read at least one byte, so ignore errors. But always
529 check for EOF, since feof may be true even though N > 0.
530 Otherwise, we could end up calling fread after EOF. */
531 if (feof (stream))
532 goto process_partial_block;
533 }
534
535 /* Process buffer with BLOCKSIZE bytes. Note that
536 BLOCKSIZE % 64 == 0
537 */
538 sha1_process_block (buffer, BLOCKSIZE, &ctx_block);
539 if (payload_offset >= 0 && payload_offset < pos)
540 {
541 /* At least part of the buffer contains data from payload. */
542 off_t start_of_payload = payload_offset - (pos - BLOCKSIZE);
543 if (start_of_payload <= 0)
544 /* All bytes in the buffer belong to the payload. */
545 start_of_payload = 0;
546
547 /* Process the payload part of the buffer.
548 Note: we can't use sha1_process_block here even if we
549 process the complete buffer. Because the payload doesn't
550 have to start with a full block, there may still be some
551 bytes left from the previous buffer. Therefore, we need
552 to continue with sha1_process_bytes. */
553 sha1_process_bytes (buffer + start_of_payload,
554 BLOCKSIZE - start_of_payload, &ctx_payload);
555 }
556 }
557
558 process_partial_block:;
559
560 /* Process any remaining bytes. */
561 if (sum > 0)
562 {
563 sha1_process_bytes (buffer, sum, &ctx_block);
564 if (payload_offset >= 0 && payload_offset < pos)
565 {
566 /* At least part of the buffer contains data from payload. */
567 off_t start_of_payload = payload_offset - (pos - sum);
568 if (start_of_payload <= 0)
569 /* All bytes in the buffer belong to the payload. */
570 start_of_payload = 0;
571
572 /* Process the payload part of the buffer. */
573 sha1_process_bytes (buffer + start_of_payload,
574 sum - start_of_payload, &ctx_payload);
575 }
576 }
577
578 /* Construct result in desired memory. */
579 sha1_finish_ctx (&ctx_block, res_block);
580 if (payload_offset >= 0)
581 sha1_finish_ctx (&ctx_payload, res_payload);
582 xfree (buffer);
583 return 0;
584
585 #undef BLOCKSIZE
586 }
587
588 /* Converts the SHA1 digest to a base32-encoded string.
589 "sha1:DIGEST\0" (Allocates a new string for the response.) */
590 static char *
591 warc_base32_sha1_digest (const char *sha1_digest, char *sha1_base32, size_t sha1_base32_size)
592 {
593 if (sha1_base32_size >= BASE32_LENGTH(SHA1_DIGEST_SIZE) + 5 + 1)
594 {
595 memcpy (sha1_base32, "sha1:", 5);
596 base32_encode (sha1_digest, SHA1_DIGEST_SIZE, sha1_base32 + 5,
597 sha1_base32_size - 5);
598 }
599 else
600 *sha1_base32 = 0;
601
602 return sha1_base32;
603 }
604
605
606 /* Sets the digest headers of the record.
607 This method will calculate the block digest and, if payload_offset >= 0,
608 will also calculate the payload digest of the payload starting at the
609 provided offset. */
610 static void
611 warc_write_digest_headers (FILE *file, long payload_offset)
612 {
613 if (opt.warc_digests_enabled)
614 {
615 /* Calculate the block and payload digests. */
616 char sha1_res_block[SHA1_DIGEST_SIZE];
617 char sha1_res_payload[SHA1_DIGEST_SIZE];
618
619 rewind (file);
620 if (warc_sha1_stream_with_payload (file, sha1_res_block,
621 sha1_res_payload, payload_offset) == 0)
622 {
623 char digest[BASE32_LENGTH(SHA1_DIGEST_SIZE) + 1 + 5];
624
625 warc_write_header ("WARC-Block-Digest",
626 warc_base32_sha1_digest (sha1_res_block, digest, sizeof(digest)));
627
628 if (payload_offset >= 0)
629 warc_write_header ("WARC-Payload-Digest",
630 warc_base32_sha1_digest (sha1_res_payload, digest, sizeof(digest)));
631 }
632 }
633 }
634
635
636 /* Fills timestamp with the current time and date.
637 The UTC time is formatted following ISO 8601, as required
638 for use in the WARC-Date header.
639 The timestamp will be 21 characters long. */
640 char *
641 warc_timestamp (char *timestamp, size_t timestamp_size)
642 {
643 time_t rawtime = time (NULL);
644 struct tm * timeinfo = gmtime (&rawtime);
645
646 if (strftime (timestamp, timestamp_size, "%Y-%m-%dT%H:%M:%SZ", timeinfo) == 0 && timestamp_size > 0)
647 *timestamp = 0;
648
649 return timestamp;
650 }
651
652 /* Fills urn_str with a UUID in the format required
653 for the WARC-Record-Id header.
654 The string will be 47 characters long. */
655 #if HAVE_LIBUUID
656 void
657 warc_uuid_str (char *urn_str, size_t urn_size)
658 {
659 char uuid_str[37];
660 uuid_t record_id;
661
662 uuid_generate (record_id);
663 uuid_unparse (record_id, uuid_str);
664
665 snprintf (urn_str, urn_size, "<urn:uuid:%s>", uuid_str);
666 }
667 #elif HAVE_UUID_CREATE
668 void
669 warc_uuid_str (char *urn_str, size_t urn_size)
670 {
671 char *uuid_str;
672 uuid_t record_id;
673
674 uuid_create (&record_id, NULL);
675 uuid_to_string (&record_id, &uuid_str, NULL);
676
677 snprintf (urn_str, urn_size, "<urn:uuid:%s>", uuid_str);
678 xfree (uuid_str);
679 }
680 #else
681 # ifdef WINDOWS
682
683 typedef RPC_STATUS (RPC_ENTRY * UuidCreate_proc) (UUID *);
684 typedef RPC_STATUS (RPC_ENTRY * UuidToString_proc) (UUID *, unsigned char **);
685 typedef RPC_STATUS (RPC_ENTRY * RpcStringFree_proc) (unsigned char **);
686
687 static int
688 windows_uuid_str (char *urn_str, size_t urn_size)
689 {
690 static UuidCreate_proc pfn_UuidCreate = NULL;
691 static UuidToString_proc pfn_UuidToString = NULL;
692 static RpcStringFree_proc pfn_RpcStringFree = NULL;
693 static int rpc_uuid_avail = -1;
694
695 /* Rpcrt4.dll is not available on older versions of Windows, so we
696 need to test its availability at run time. */
697 if (rpc_uuid_avail == -1)
698 {
699 HMODULE hm_rpcrt4 = LoadLibrary ("Rpcrt4.dll");
700
701 if (hm_rpcrt4)
702 {
703 pfn_UuidCreate =
704 (UuidCreate_proc) GetProcAddress (hm_rpcrt4, "UuidCreate");
705 pfn_UuidToString =
706 (UuidToString_proc) GetProcAddress (hm_rpcrt4, "UuidToStringA");
707 pfn_RpcStringFree =
708 (RpcStringFree_proc) GetProcAddress (hm_rpcrt4, "RpcStringFreeA");
709 if (pfn_UuidCreate && pfn_UuidToString && pfn_RpcStringFree)
710 rpc_uuid_avail = 1;
711 else
712 rpc_uuid_avail = 0;
713 }
714 else
715 rpc_uuid_avail = 0;
716 }
717
718 if (rpc_uuid_avail)
719 {
720 BYTE *uuid_str;
721 UUID uuid;
722
723 if (pfn_UuidCreate (&uuid) == RPC_S_OK)
724 {
725 if (pfn_UuidToString (&uuid, &uuid_str) == RPC_S_OK)
726 {
727 snprintf (urn_str, urn_size, "<urn:uuid:%s>", uuid_str);
728 pfn_RpcStringFree (&uuid_str);
729 return 1;
730 }
731 }
732 }
733 return 0;
734 }
735 #endif
736 /* Fills urn_str with a UUID based on random numbers in the format
737 required for the WARC-Record-Id header.
738 (See RFC 4122, UUID version 4.)
739
740 Note: this is a fallback method, it is much better to use the
741 methods provided by libuuid.
742
743 The string will be 47 characters long. */
744 void
745 warc_uuid_str (char *urn_str, size_t urn_size)
746 {
747 /* RFC 4122, a version 4 UUID with only random numbers */
748
749 unsigned char uuid_data[16];
750 int i;
751
752 #ifdef WINDOWS
753 /* If the native method fails (expected on older Windows versions),
754 use the fallback below. */
755 if (windows_uuid_str (urn_str, urn_size))
756 return;
757 #endif
758
759 for (i=0; i<16; i++)
760 uuid_data[i] = random_number (255);
761
762 /* Set the four most significant bits (bits 12 through 15) of the
763 * time_hi_and_version field to the 4-bit version number */
764 uuid_data[6] = (uuid_data[6] & 0x0F) | 0x40;
765
766 /* Set the two most significant bits (bits 6 and 7) of the
767 * clock_seq_hi_and_reserved to zero and one, respectively. */
768 uuid_data[8] = (uuid_data[8] & 0xBF) | 0x80;
769
770 snprintf (urn_str, urn_size,
771 "<urn:uuid:%02x%02x%02x%02x-%02x%02x-%02x%02x-%02x%02x-%02x%02x%02x%02x%02x%02x>",
772 uuid_data[0], uuid_data[1], uuid_data[2], uuid_data[3], uuid_data[4],
773 uuid_data[5], uuid_data[6], uuid_data[7], uuid_data[8], uuid_data[9],
774 uuid_data[10], uuid_data[11], uuid_data[12], uuid_data[13], uuid_data[14],
775 uuid_data[15]);
776 }
777 #endif
778
779 /* Write a warcinfo record to the current file.
780 Updates warc_current_warcinfo_uuid_str. */
781 static bool
782 warc_write_warcinfo_record (const char *filename)
783 {
784 FILE *warc_tmp;
785 char timestamp[22];
786 char *filename_basename;
787
788 /* Write warc-info record as the first record of the file. */
789 /* We add the record id of this info record to the other records in the
790 file. */
791 warc_uuid_str (warc_current_warcinfo_uuid_str, sizeof (warc_current_warcinfo_uuid_str));
792
793 warc_timestamp (timestamp, sizeof(timestamp));
794
795 filename_basename = base_name (filename);
796
797 warc_write_start_record ();
798 warc_write_header ("WARC-Type", "warcinfo");
799 warc_write_header ("Content-Type", "application/warc-fields");
800 warc_write_header ("WARC-Date", timestamp);
801 warc_write_header ("WARC-Record-ID", warc_current_warcinfo_uuid_str);
802 warc_write_header ("WARC-Filename", filename_basename);
803
804 xfree (filename_basename);
805
806 /* Create content. */
807 warc_tmp = warc_tempfile ();
808 if (warc_tmp == NULL)
809 {
810 return false;
811 }
812
813 fprintf (warc_tmp, "software: Wget/%s (%s)\r\n", version_string, OS_TYPE);
814 fprintf (warc_tmp, "format: WARC File Format 1.0\r\n");
815 fprintf (warc_tmp,
816 "conformsTo: http://bibnum.bnf.fr/WARC/WARC_ISO_28500_version1_latestdraft.pdf\r\n");
817 fprintf (warc_tmp, "robots: %s\r\n", (opt.use_robots ? "classic" : "off"));
818 fprintf (warc_tmp, "wget-arguments: %s\r\n", program_argstring);
819 /* Add the user headers, if any. */
820 if (opt.warc_user_headers)
821 {
822 int i;
823 for (i = 0; opt.warc_user_headers[i]; i++)
824 fprintf (warc_tmp, "%s\r\n", opt.warc_user_headers[i]);
825 }
826 fprintf(warc_tmp, "\r\n");
827
828 warc_write_digest_headers (warc_tmp, -1);
829 warc_write_block_from_file (warc_tmp);
830 warc_write_end_record ();
831
832 if (! warc_write_ok)
833 logprintf (LOG_NOTQUIET, _("Error writing warcinfo record to WARC file.\n"));
834
835 fclose (warc_tmp);
836 return warc_write_ok;
837 }
838
839 /* Opens a new WARC file.
840 If META is true, generates a filename ending with 'meta.warc.gz'.
841
842 This method will:
843 1. close the current WARC file (if there is one);
844 2. increment warc_current_file_number;
845 3. open a new WARC file;
846 4. write the initial warcinfo record.
847
848 Returns true on success, false otherwise.
849 */
850 static bool
851 warc_start_new_file (bool meta)
852 {
853 #ifdef __VMS
854 # define WARC_GZ "warc-gz"
855 #else /* def __VMS */
856 # define WARC_GZ "warc.gz"
857 #endif /* def __VMS [else] */
858
859 #ifdef HAVE_LIBZ
860 const char *extension = (opt.warc_compression_enabled ? WARC_GZ : "warc");
861 #else
862 const char *extension = "warc";
863 #endif
864
865 if (opt.warc_filename == NULL)
866 return false;
867
868 if (warc_current_file != NULL)
869 fclose (warc_current_file);
870
871 *warc_current_warcinfo_uuid_str = 0;
872 xfree (warc_current_filename);
873
874 warc_current_file_number++;
875
876 /* If max size is enabled, we add a serial number to the file names. */
877 if (meta)
878 warc_current_filename = aprintf ("%s-meta.%s", opt.warc_filename, extension);
879 else if (opt.warc_maxsize > 0)
880 warc_current_filename = aprintf ("%s-%05d.%s", opt.warc_filename, warc_current_file_number, extension);
881 else
882 warc_current_filename = aprintf ("%s.%s", opt.warc_filename, extension);
883
884 logprintf (LOG_VERBOSE, _("Opening WARC file %s.\n\n"), quote (warc_current_filename));
885
886 /* Open the WARC file. */
887 warc_current_file = fopen (warc_current_filename, "wb+");
888 if (warc_current_file == NULL)
889 {
890 logprintf (LOG_NOTQUIET, _("Error opening WARC file %s.\n"),
891 quote (warc_current_filename));
892 return false;
893 }
894
895 if (! warc_write_warcinfo_record (warc_current_filename))
896 return false;
897
898 /* Add warcinfo uuid to manifest. */
899 if (warc_manifest_fp)
900 fprintf (warc_manifest_fp, "%s\n", warc_current_warcinfo_uuid_str);
901
902 return true;
903 }
904
905 /* Opens the CDX file for output. */
906 static bool
907 warc_start_cdx_file (void)
908 {
909 char *cdx_filename = aprintf("%s.cdx", opt.warc_filename);
910 warc_current_cdx_file = fopen (cdx_filename, "a+");
911 free(cdx_filename);
912
913 if (warc_current_cdx_file == NULL)
914 return false;
915
916 /* Print the CDX header.
917 *
918 * a - original url
919 * b - date
920 * m - mime type
921 * s - response code
922 * k - new style checksum
923 * r - redirect
924 * M - meta tags
925 * V - compressed arc file offset
926 * g - file name
927 * u - record-id
928 */
929 fprintf (warc_current_cdx_file, " CDX a b a m s k r M V g u\n");
930 fflush (warc_current_cdx_file);
931
932 return true;
933 }
934
935 #define CDX_FIELDSEP " \t\r\n"
936
937 /* Parse the CDX header and find the field numbers of the original url,
938 checksum and record ID fields. */
939 static bool
940 warc_parse_cdx_header (char *lineptr, int *field_num_original_url,
941 int *field_num_checksum, int *field_num_record_id)
942 {
943 char *token;
944 char *save_ptr;
945
946 *field_num_original_url = -1;
947 *field_num_checksum = -1;
948 *field_num_record_id = -1;
949
950 token = strtok_r (lineptr, CDX_FIELDSEP, &save_ptr);
951
952 if (token != NULL && strcmp (token, "CDX") == 0)
953 {
954 int field_num = 0;
955 while (token != NULL)
956 {
957 token = strtok_r (NULL, CDX_FIELDSEP, &save_ptr);
958 if (token != NULL)
959 {
960 switch (token[0])
961 {
962 case 'a':
963 *field_num_original_url = field_num;
964 break;
965 case 'k':
966 *field_num_checksum = field_num;
967 break;
968 case 'u':
969 *field_num_record_id = field_num;
970 break;
971 }
972 }
973 field_num++;
974 }
975 }
976
977 return *field_num_original_url != -1
978 && *field_num_checksum != -1
979 && *field_num_record_id != -1;
980 }
981
982 /* Parse the CDX record and add it to the warc_cdx_dedup_table hash table. */
983 static void
984 warc_process_cdx_line (char *lineptr, int field_num_original_url,
985 int field_num_checksum, int field_num_record_id)
986 {
987 char *original_url = NULL;
988 char *checksum = NULL;
989 char *record_id = NULL;
990 char *token;
991 char *save_ptr;
992 int field_num = 0;
993
994 /* Read this line to get the fields we need. */
995 token = strtok_r (lineptr, CDX_FIELDSEP, &save_ptr);
996 while (token != NULL)
997 {
998 char **val;
999 if (field_num == field_num_original_url)
1000 val = &original_url;
1001 else if (field_num == field_num_checksum)
1002 val = &checksum;
1003 else if (field_num == field_num_record_id)
1004 val = &record_id;
1005 else
1006 val = NULL;
1007
1008 if (val != NULL)
1009 *val = strdup (token);
1010
1011 token = strtok_r (NULL, CDX_FIELDSEP, &save_ptr);
1012 field_num++;
1013 }
1014
1015 if (original_url != NULL && checksum != NULL && record_id != NULL)
1016 {
1017 /* For some extra efficiency, we decode the base32 encoded
1018 checksum value. This should produce exactly SHA1_DIGEST_SIZE
1019 bytes. */
1020 idx_t checksum_l;
1021 char * checksum_v;
1022 base32_decode_alloc (checksum, strlen (checksum), &checksum_v,
1023 &checksum_l);
1024 xfree (checksum);
1025
1026 if (checksum_v != NULL && checksum_l == SHA1_DIGEST_SIZE)
1027 {
1028 /* This is a valid line with a valid checksum. */
1029 struct warc_cdx_record *rec;
1030 rec = xmalloc (sizeof (struct warc_cdx_record));
1031 rec->url = original_url;
1032 rec->uuid = record_id;
1033 memcpy (rec->digest, checksum_v, SHA1_DIGEST_SIZE);
1034 hash_table_put (warc_cdx_dedup_table, rec->digest, rec);
1035 xfree (checksum_v);
1036 }
1037 else
1038 {
1039 xfree (original_url);
1040 xfree (checksum_v);
1041 xfree (record_id);
1042 }
1043 }
1044 else
1045 {
1046 xfree(checksum);
1047 xfree(original_url);
1048 xfree(record_id);
1049 }
1050 }
1051
1052 /* Loads the CDX file from opt.warc_cdx_dedup_filename and fills
1053 the warc_cdx_dedup_table. */
1054 static bool
1055 warc_load_cdx_dedup_file (void)
1056 {
1057 FILE *f;
1058 char *lineptr = NULL;
1059 size_t n = 0;
1060 ssize_t line_length;
1061 int field_num_original_url = -1;
1062 int field_num_checksum = -1;
1063 int field_num_record_id = -1;
1064
1065 f = fopen (opt.warc_cdx_dedup_filename, "r");
1066 if (f == NULL)
1067 return false;
1068
1069 /* The first line should contain the CDX header.
1070 Format: " CDX x x x x x"
1071 where x are field type indicators. For our purposes, we only
1072 need 'a' (the original url), 'k' (the SHA1 checksum) and
1073 'u' (the WARC record id). */
1074 line_length = getline (&lineptr, &n, f);
1075 if (line_length != -1)
1076 warc_parse_cdx_header (lineptr, &field_num_original_url,
1077 &field_num_checksum, &field_num_record_id);
1078
1079 /* If the file contains all three fields, read the complete file. */
1080 if (field_num_original_url == -1
1081 || field_num_checksum == -1
1082 || field_num_record_id == -1)
1083 {
1084 if (field_num_original_url == -1)
1085 logprintf (LOG_NOTQUIET,
1086 _("CDX file does not list original urls. (Missing column 'a'.)\n"));
1087 if (field_num_checksum == -1)
1088 logprintf (LOG_NOTQUIET,
1089 _("CDX file does not list checksums. (Missing column 'k'.)\n"));
1090 if (field_num_record_id == -1)
1091 logprintf (LOG_NOTQUIET,
1092 _("CDX file does not list record ids. (Missing column 'u'.)\n"));
1093 }
1094 else
1095 {
1096 int nrecords;
1097
1098 /* Initialize the table. */
1099 warc_cdx_dedup_table = hash_table_new (1000, warc_hash_sha1_digest,
1100 warc_cmp_sha1_digest);
1101
1102 do
1103 {
1104 line_length = getline (&lineptr, &n, f);
1105 if (line_length != -1)
1106 {
1107 warc_process_cdx_line (lineptr, field_num_original_url,
1108 field_num_checksum, field_num_record_id);
1109 }
1110
1111 }
1112 while (line_length != -1);
1113
1114 /* Print results. */
1115 nrecords = hash_table_count (warc_cdx_dedup_table);
1116 logprintf (LOG_VERBOSE, ngettext ("Loaded %d record from CDX.\n\n",
1117 "Loaded %d records from CDX.\n\n",
1118 nrecords),
1119 nrecords);
1120 }
1121
1122 xfree (lineptr);
1123 fclose (f);
1124
1125 return true;
1126 }
1127 #undef CDX_FIELDSEP
1128
1129 /* Returns the existing duplicate CDX record for the given url and payload
1130 digest. Returns NULL if the url is not found or if the payload digest
1131 does not match, or if CDX deduplication is disabled. */
1132 static struct warc_cdx_record *
1133 warc_find_duplicate_cdx_record (const char *url, char *sha1_digest_payload)
1134 {
1135 struct warc_cdx_record *rec_existing;
1136
1137 if (warc_cdx_dedup_table == NULL)
1138 return NULL;
1139
1140 rec_existing = hash_table_get (warc_cdx_dedup_table, sha1_digest_payload);
1141
1142 if (rec_existing && strcmp (rec_existing->url, url) == 0)
1143 return rec_existing;
1144 else
1145 return NULL;
1146 }
1147
1148 /* Initializes the WARC writer (if opt.warc_filename is set).
1149 This should be called before any WARC record is written. */
1150 void
1151 warc_init (void)
1152 {
1153 warc_write_ok = true;
1154
1155 if (opt.warc_filename != NULL)
1156 {
1157 if (opt.warc_cdx_dedup_filename != NULL)
1158 {
1159 if (! warc_load_cdx_dedup_file ())
1160 {
1161 logprintf (LOG_NOTQUIET,
1162 _("Could not read CDX file %s for deduplication.\n"),
1163 quote (opt.warc_cdx_dedup_filename));
1164 exit (WGET_EXIT_GENERIC_ERROR);
1165 }
1166 }
1167
1168 warc_manifest_fp = warc_tempfile ();
1169 if (warc_manifest_fp == NULL)
1170 {
1171 logprintf (LOG_NOTQUIET,
1172 _("Could not open temporary WARC manifest file.\n"));
1173 exit (WGET_EXIT_GENERIC_ERROR);
1174 }
1175
1176 if (opt.warc_keep_log)
1177 {
1178 warc_log_fp = warc_tempfile ();
1179 if (warc_log_fp == NULL)
1180 {
1181 logprintf (LOG_NOTQUIET,
1182 _("Could not open temporary WARC log file.\n"));
1183 exit (WGET_EXIT_GENERIC_ERROR);
1184 }
1185 log_set_warc_log_fp (warc_log_fp);
1186 }
1187
1188 warc_current_file_number = -1;
1189 if (! warc_start_new_file (false))
1190 {
1191 logprintf (LOG_NOTQUIET, _("Could not open WARC file.\n"));
1192 exit (WGET_EXIT_GENERIC_ERROR);
1193 }
1194
1195 if (opt.warc_cdx_enabled)
1196 {
1197 if (! warc_start_cdx_file ())
1198 {
1199 logprintf (LOG_NOTQUIET,
1200 _("Could not open CDX file for output.\n"));
1201 exit (WGET_EXIT_GENERIC_ERROR);
1202 }
1203 }
1204 }
1205 }
1206
1207 /* Writes metadata (manifest, configuration, log file) to the WARC file. */
1208 static void
1209 warc_write_metadata (void)
1210 {
1211 char manifest_uuid[48];
1212 FILE *warc_tmp_fp;
1213
1214 /* If there are multiple WARC files, the metadata should be written to a separate file. */
1215 if (opt.warc_maxsize > 0)
1216 warc_start_new_file (true);
1217
1218 warc_uuid_str (manifest_uuid, sizeof (manifest_uuid));
1219
1220 fflush (warc_manifest_fp);
1221 warc_write_metadata_record (manifest_uuid,
1222 "metadata://gnu.org/software/wget/warc/MANIFEST.txt",
1223 NULL, NULL, NULL, "text/plain",
1224 warc_manifest_fp, -1);
1225 /* warc_write_resource_record has closed warc_manifest_fp. */
1226
1227 warc_tmp_fp = warc_tempfile ();
1228 if (warc_tmp_fp == NULL)
1229 {
1230 logprintf (LOG_NOTQUIET, _("Could not open temporary WARC file.\n"));
1231 exit (WGET_EXIT_GENERIC_ERROR);
1232 }
1233 fflush (warc_tmp_fp);
1234 fprintf (warc_tmp_fp, "%s\n", program_argstring);
1235
1236 warc_write_resource_record (NULL,
1237 "metadata://gnu.org/software/wget/warc/wget_arguments.txt",
1238 NULL, manifest_uuid, NULL, "text/plain",
1239 warc_tmp_fp, -1);
1240 /* warc_write_resource_record has closed warc_tmp_fp. */
1241
1242 if (warc_log_fp != NULL)
1243 {
1244 warc_write_resource_record (NULL,
1245 "metadata://gnu.org/software/wget/warc/wget.log",
1246 NULL, manifest_uuid, NULL, "text/plain",
1247 warc_log_fp, -1);
1248 /* warc_write_resource_record has closed warc_log_fp. */
1249
1250 warc_log_fp = NULL;
1251 log_set_warc_log_fp (NULL);
1252 }
1253 }
1254
1255 /* Finishes the WARC writing.
1256 This should be called at the end of the program. */
1257 void
1258 warc_close (void)
1259 {
1260 if (warc_current_file != NULL)
1261 {
1262 warc_write_metadata ();
1263 *warc_current_warcinfo_uuid_str = 0;
1264 fclose (warc_current_file);
1265 warc_current_file = NULL;
1266 }
1267
1268 if (warc_current_cdx_file != NULL)
1269 {
1270 fclose (warc_current_cdx_file);
1271 warc_current_cdx_file = NULL;
1272 }
1273
1274 if (warc_log_fp != NULL)
1275 {
1276 fclose (warc_log_fp);
1277 log_set_warc_log_fp (NULL);
1278 }
1279
1280 xfree (warc_current_filename);
1281 }
1282
1283 /* Creates a temporary file for writing WARC output.
1284 The temporary file will be created in opt.warc_tempdir.
1285 Returns the pointer to the temporary file, or NULL. */
1286 FILE *
1287 warc_tempfile (void)
1288 {
1289 char filename[100];
1290 int fd;
1291
1292 if (path_search (filename, 100, opt.warc_tempdir, "wget", true) == -1)
1293 return NULL;
1294
1295 #ifdef __VMS
1296 /* 2013-07-12 SMS.
1297 * mkostemp()+unlink()+fdopen() scheme causes trouble on VMS, so use
1298 * mktemp() to uniquify the (VMS-style) name, and then use a normal
1299 * fopen() with a "create temp file marked for delete" option.
1300 */
1301 {
1302 char *tfn;
1303
1304 tfn = mktemp (filename); /* Get unique name from template. */
1305 if (tfn == NULL)
1306 return NULL;
1307 return fopen (tfn, "w+", "fop=tmd"); /* Create auto-delete temp file. */
1308 }
1309 #else /* def __VMS */
1310 fd = mkostemp (filename, O_TEMPORARY);
1311 if (fd < 0)
1312 return NULL;
1313
1314 #if !O_TEMPORARY
1315 if (unlink (filename) < 0)
1316 {
1317 close(fd);
1318 return NULL;
1319 }
1320 #endif
1321
1322 return fdopen (fd, "wb+");
1323 #endif /* def __VMS [else] */
1324 }
1325
1326
1327 /* Writes a request record to the WARC file.
1328 url is the target uri of the request,
1329 timestamp_str is the timestamp of the request (generated with warc_timestamp),
1330 record_uuid is the uuid of the request (generated with warc_uuid_str),
1331 body is a pointer to a file containing the request headers and body.
1332 ip is the ip address of the server (or NULL),
1333 Calling this function will close body.
1334 Returns true on success, false on error. */
1335 bool
1336 warc_write_request_record (const char *url, const char *timestamp_str,
1337 const char *record_uuid, const ip_address *ip,
1338 FILE *body, off_t payload_offset)
1339 {
1340 warc_write_start_record ();
1341 warc_write_header ("WARC-Type", "request");
1342 warc_write_header_uri ("WARC-Target-URI", url);
1343 warc_write_header ("Content-Type", "application/http;msgtype=request");
1344 warc_write_date_header (timestamp_str);
1345 warc_write_header ("WARC-Record-ID", record_uuid);
1346 warc_write_ip_header (ip);
1347 warc_write_header ("WARC-Warcinfo-ID", warc_current_warcinfo_uuid_str);
1348 warc_write_digest_headers (body, payload_offset);
1349 warc_write_block_from_file (body);
1350 warc_write_end_record ();
1351
1352 fclose (body);
1353
1354 return warc_write_ok;
1355 }
1356
1357 /* Writes a response record to the CDX file.
1358 url is the target uri of the request/response,
1359 timestamp_str is the timestamp of the request that generated this response,
1360 (generated with warc_timestamp),
1361 mime_type is the mime type of the response body (will be printed to CDX),
1362 response_code is the HTTP response code (will be printed to CDX),
1363 payload_digest is the sha1 digest of the payload,
1364 redirect_location is the contents of the Location: header, or NULL (will be printed to CDX),
1365 offset is the position of the WARC record in the WARC file,
1366 warc_filename is the filename of the WARC,
1367 response_uuid is the uuid of the response.
1368 Returns true on success, false on error. */
1369 static bool
1370 warc_write_cdx_record (const char *url, const char *timestamp_str,
1371 const char *mime_type, int response_code,
1372 const char *payload_digest, const char *redirect_location,
1373 off_t offset, const char *warc_filename _GL_UNUSED,
1374 const char *response_uuid)
1375 {
1376 /* Transform the timestamp. */
1377 char timestamp_str_cdx[15];
1378 char offset_string[MAX_INT_TO_STRING_LEN(off_t)];
1379 const char *checksum;
1380 char *tmp_location = NULL;
1381
1382 memcpy (timestamp_str_cdx , timestamp_str , 4); /* "YYYY" "-" */
1383 memcpy (timestamp_str_cdx + 4, timestamp_str + 5, 2); /* "mm" "-" */
1384 memcpy (timestamp_str_cdx + 6, timestamp_str + 8, 2); /* "dd" "T" */
1385 memcpy (timestamp_str_cdx + 8, timestamp_str + 11, 2); /* "HH" ":" */
1386 memcpy (timestamp_str_cdx + 10, timestamp_str + 14, 2); /* "MM" ":" */
1387 memcpy (timestamp_str_cdx + 12, timestamp_str + 17, 2); /* "SS" "Z" */
1388 timestamp_str_cdx[14] = '\0';
1389
1390 /* Rewrite the checksum. */
1391 if (payload_digest != NULL)
1392 checksum = payload_digest + 5; /* Skip the "sha1:" */
1393 else
1394 checksum = "-";
1395
1396 if (mime_type == NULL || strlen(mime_type) == 0)
1397 mime_type = "-";
1398 if (redirect_location == NULL || strlen(redirect_location) == 0)
1399 tmp_location = strdup ("-");
1400 else
1401 tmp_location = url_escape(redirect_location);
1402
1403 number_to_string (offset_string, offset);
1404
1405 /* Print the CDX line. */
1406 fprintf (warc_current_cdx_file, "%s %s %s %s %d %s %s - %s %s %s\n", url,
1407 timestamp_str_cdx, url, mime_type, response_code, checksum,
1408 tmp_location, offset_string, warc_current_filename,
1409 response_uuid);
1410 fflush (warc_current_cdx_file);
1411 free (tmp_location);
1412
1413 return true;
1414 }
1415
1416 /* Writes a revisit record to the WARC file.
1417 url is the target uri of the request/response,
1418 timestamp_str is the timestamp of the request that generated this response
1419 (generated with warc_timestamp),
1420 concurrent_to_uuid is the uuid of the request for that generated this response
1421 (generated with warc_uuid_str),
1422 refers_to_uuid is the uuid of the original response
1423 (generated with warc_uuid_str),
1424 payload_digest is the sha1 digest of the payload,
1425 ip is the ip address of the server (or NULL),
1426 body is a pointer to a file containing the response headers (without payload).
1427 Calling this function will close body.
1428 Returns true on success, false on error. */
1429 static bool
1430 warc_write_revisit_record (const char *url, const char *timestamp_str,
1431 const char *concurrent_to_uuid, const char *payload_digest,
1432 const char *refers_to, const ip_address *ip, FILE *body)
1433 {
1434 char revisit_uuid [48];
1435 char block_digest[BASE32_LENGTH(SHA1_DIGEST_SIZE) + 1 + 5];
1436 char sha1_res_block[SHA1_DIGEST_SIZE];
1437
1438 warc_uuid_str (revisit_uuid, sizeof (revisit_uuid));
1439
1440 sha1_stream (body, sha1_res_block);
1441 warc_base32_sha1_digest (sha1_res_block, block_digest, sizeof(block_digest));
1442
1443 warc_write_start_record ();
1444 warc_write_header ("WARC-Type", "revisit");
1445 warc_write_header ("WARC-Record-ID", revisit_uuid);
1446 warc_write_header ("WARC-Warcinfo-ID", warc_current_warcinfo_uuid_str);
1447 warc_write_header ("WARC-Concurrent-To", concurrent_to_uuid);
1448 warc_write_header ("WARC-Refers-To", refers_to);
1449 warc_write_header ("WARC-Profile", "http://netpreserve.org/warc/1.0/revisit/identical-payload-digest");
1450 warc_write_header ("WARC-Truncated", "length");
1451 warc_write_header_uri ("WARC-Target-URI", url);
1452 warc_write_date_header (timestamp_str);
1453 warc_write_ip_header (ip);
1454 warc_write_header ("Content-Type", "application/http;msgtype=response");
1455 warc_write_header ("WARC-Block-Digest", block_digest);
1456 warc_write_header ("WARC-Payload-Digest", payload_digest);
1457 warc_write_block_from_file (body);
1458 warc_write_end_record ();
1459
1460 fclose (body);
1461
1462 return warc_write_ok;
1463 }
1464
1465 /* Writes a response record to the WARC file.
1466 url is the target uri of the request/response,
1467 timestamp_str is the timestamp of the request that generated this response
1468 (generated with warc_timestamp),
1469 concurrent_to_uuid is the uuid of the request for that generated this response
1470 (generated with warc_uuid_str),
1471 ip is the ip address of the server (or NULL),
1472 body is a pointer to a file containing the response headers and body.
1473 mime_type is the mime type of the response body (will be printed to CDX),
1474 response_code is the HTTP response code (will be printed to CDX),
1475 redirect_location is the contents of the Location: header, or NULL (will be printed to CDX),
1476 Calling this function will close body.
1477 Returns true on success, false on error. */
1478 bool
1479 warc_write_response_record (const char *url, const char *timestamp_str,
1480 const char *concurrent_to_uuid, const ip_address *ip,
1481 FILE *body, off_t payload_offset, const char *mime_type,
1482 int response_code, const char *redirect_location)
1483 {
1484 char block_digest[BASE32_LENGTH(SHA1_DIGEST_SIZE) + 1 + 5];
1485 char payload_digest[BASE32_LENGTH(SHA1_DIGEST_SIZE) + 1 + 5];
1486 char sha1_res_block[SHA1_DIGEST_SIZE];
1487 char sha1_res_payload[SHA1_DIGEST_SIZE];
1488 char response_uuid [48];
1489 off_t offset;
1490
1491 if (opt.warc_digests_enabled)
1492 {
1493 /* Calculate the block and payload digests. */
1494 rewind (body);
1495 if (warc_sha1_stream_with_payload (body, sha1_res_block, sha1_res_payload,
1496 payload_offset) == 0)
1497 {
1498 /* Decide (based on url + payload digest) if we have seen this
1499 data before. */
1500 struct warc_cdx_record *rec_existing;
1501 rec_existing = warc_find_duplicate_cdx_record (url, sha1_res_payload);
1502 if (rec_existing != NULL)
1503 {
1504 bool result;
1505
1506 /* Found an existing record. */
1507 logprintf (LOG_VERBOSE,
1508 _("Found exact match in CDX file. Saving revisit record to WARC.\n"));
1509
1510 /* Remove the payload from the file. */
1511 if (payload_offset > 0)
1512 {
1513 if (ftruncate (fileno (body), payload_offset) == -1)
1514 return false;
1515 }
1516
1517 /* Send the original payload digest. */
1518 warc_base32_sha1_digest (sha1_res_payload, payload_digest, sizeof(payload_digest));
1519 result = warc_write_revisit_record (url, timestamp_str,
1520 concurrent_to_uuid, payload_digest, rec_existing->uuid,
1521 ip, body);
1522
1523 return result;
1524 }
1525
1526 warc_base32_sha1_digest (sha1_res_block, block_digest, sizeof(block_digest));
1527 warc_base32_sha1_digest (sha1_res_payload, payload_digest, sizeof(payload_digest));
1528 }
1529 }
1530
1531 /* Not a revisit, just store the record. */
1532
1533 warc_uuid_str (response_uuid, sizeof (response_uuid));
1534
1535 fseeko (warc_current_file, 0L, SEEK_END);
1536 offset = ftello (warc_current_file);
1537
1538 warc_write_start_record ();
1539 warc_write_header ("WARC-Type", "response");
1540 warc_write_header ("WARC-Record-ID", response_uuid);
1541 warc_write_header ("WARC-Warcinfo-ID", warc_current_warcinfo_uuid_str);
1542 warc_write_header ("WARC-Concurrent-To", concurrent_to_uuid);
1543 warc_write_header_uri ("WARC-Target-URI", url);
1544 warc_write_date_header (timestamp_str);
1545 warc_write_ip_header (ip);
1546 warc_write_header ("WARC-Block-Digest", block_digest);
1547 warc_write_header ("WARC-Payload-Digest", payload_digest);
1548 warc_write_header ("Content-Type", "application/http;msgtype=response");
1549 warc_write_block_from_file (body);
1550 warc_write_end_record ();
1551
1552 fclose (body);
1553
1554 if (warc_write_ok && opt.warc_cdx_enabled)
1555 {
1556 /* Add this record to the CDX. */
1557 warc_write_cdx_record (url, timestamp_str, mime_type, response_code,
1558 payload_digest, redirect_location, offset, warc_current_filename,
1559 response_uuid);
1560 }
1561
1562 return warc_write_ok;
1563 }
1564
1565 /* Writes a resource or metadata record to the WARC file.
1566 warc_type is either "resource" or "metadata",
1567 resource_uuid is the uuid of the resource (or NULL),
1568 url is the target uri of the resource,
1569 timestamp_str is the timestamp (generated with warc_timestamp),
1570 concurrent_to_uuid is the uuid of the record that generated this,
1571 resource (generated with warc_uuid_str) or NULL,
1572 ip is the ip address of the server (or NULL),
1573 content_type is the mime type of the body (or NULL),
1574 body is a pointer to a file containing the resource data.
1575 Calling this function will close body.
1576 Returns true on success, false on error. */
1577 static bool
1578 warc_write_record (const char *record_type, const char *resource_uuid,
1579 const char *url, const char *timestamp_str,
1580 const char *concurrent_to_uuid,
1581 const ip_address *ip, const char *content_type, FILE *body,
1582 off_t payload_offset)
1583 {
1584 char uuid_buf[48];
1585
1586 if (resource_uuid == NULL)
1587 {
1588 warc_uuid_str (uuid_buf, sizeof (uuid_buf));
1589 resource_uuid = uuid_buf;
1590 }
1591
1592 if (content_type == NULL)
1593 content_type = "application/octet-stream";
1594
1595 warc_write_start_record ();
1596 warc_write_header ("WARC-Type", record_type);
1597 warc_write_header ("WARC-Record-ID", resource_uuid);
1598 warc_write_header ("WARC-Warcinfo-ID", warc_current_warcinfo_uuid_str);
1599 warc_write_header ("WARC-Concurrent-To", concurrent_to_uuid);
1600 warc_write_header_uri ("WARC-Target-URI", url);
1601 warc_write_date_header (timestamp_str);
1602 warc_write_ip_header (ip);
1603 warc_write_digest_headers (body, payload_offset);
1604 warc_write_header ("Content-Type", content_type);
1605 warc_write_block_from_file (body);
1606 warc_write_end_record ();
1607
1608 fclose (body);
1609
1610 return warc_write_ok;
1611 }
1612
1613 /* Writes a resource record to the WARC file.
1614 resource_uuid is the uuid of the resource (or NULL),
1615 url is the target uri of the resource,
1616 timestamp_str is the timestamp (generated with warc_timestamp),
1617 concurrent_to_uuid is the uuid of the record that generated this,
1618 resource (generated with warc_uuid_str) or NULL,
1619 ip is the ip address of the server (or NULL),
1620 content_type is the mime type of the body (or NULL),
1621 body is a pointer to a file containing the resource data.
1622 Calling this function will close body.
1623 Returns true on success, false on error. */
1624 bool
1625 warc_write_resource_record (const char *resource_uuid, const char *url,
1626 const char *timestamp_str, const char *concurrent_to_uuid,
1627 const ip_address *ip, const char *content_type, FILE *body,
1628 off_t payload_offset)
1629 {
1630 return warc_write_record ("resource",
1631 resource_uuid, url, timestamp_str, concurrent_to_uuid,
1632 ip, content_type, body, payload_offset);
1633 }
1634
1635 /* Writes a metadata record to the WARC file.
1636 record_uuid is the uuid of the record (or NULL),
1637 url is the target uri of the record,
1638 timestamp_str is the timestamp (generated with warc_timestamp),
1639 concurrent_to_uuid is the uuid of the record that generated this,
1640 record (generated with warc_uuid_str) or NULL,
1641 ip is the ip address of the server (or NULL),
1642 content_type is the mime type of the body (or NULL),
1643 body is a pointer to a file containing the record data.
1644 Calling this function will close body.
1645 Returns true on success, false on error. */
1646 bool
1647 warc_write_metadata_record (const char *record_uuid, const char *url,
1648 const char *timestamp_str, const char *concurrent_to_uuid,
1649 ip_address *ip, const char *content_type, FILE *body,
1650 off_t payload_offset)
1651 {
1652 return warc_write_record ("metadata",
1653 record_uuid, url, timestamp_str, concurrent_to_uuid,
1654 ip, content_type, body, payload_offset);
1655 }