This is a crude patch for crawl to log all http headers received.
See http://www.monkey.org/~provos/crawl/

  --md@hudora.de


--- crawl-0.4/http.c	2003-05-18 03:50:24.000000000 +0200
+++ crawl-0.4-md/http.c	2004-10-02 02:57:24.000000000 +0200
@@ -77,6 +77,8 @@
 char *rawname = NULL;
 char *statefile = NULL;
 
+FILE *fpheaders = NULL; 
+
 void (*http_movecb)(struct uri *, char *) = NULL;
 
 /* Timeout used for postevent */
@@ -249,6 +251,10 @@
 		}
 	}
 	
+
+        if((fpheaders = fopen("headers.log", "a")) == NULL)
+		err(1, "fopen");
+
 	return (0);
 }
 
@@ -694,6 +700,8 @@
 
 	fclose(fpstate);
 
+	fclose(fpheaders);
+
 	fprintf(stdout, "Terminated with %d saved urls.\n", linenum);
 }
 
@@ -1099,6 +1107,8 @@
 		uri->header[offset] = '\0';
 	}
 
+	fprintf(fpheaders, "http://%s%s\n",	uri->url_host, uri->url_file);
+	fprintf(fpheaders, "%s\n", uri->header);
 	http_parseheader(uri);
 	return;