[pacman-dev] [PATCH] support for mtime checking @ http database download
Tino Reichardt
list-pacman-dev at mcmilk.de
Wed Sep 6 13:25:27 EDT 2006
Hello list,
I have done a small patch, which adds mtime checking for the
database download on http archives...
overview:
- one new function called HttpGetMod() in ftplib.c
- some new if's @ pacsync.c
I think there may be a need for it ;)
--
regards, TR
-------------- next part --------------
diff -urN pacman-2.9.8.orig/libftp/ftplib.c pacman-2.9.8/libftp/ftplib.c
--- pacman-2.9.8.orig/libftp/ftplib.c 2004-07-03 06:25:47.000000000 +0200
+++ pacman-2.9.8/libftp/ftplib.c 2006-09-06 19:23:11.000000000 +0200
@@ -1556,6 +1556,103 @@
}
/*
+ * Convert the textual specification of time in TIME_STRING to the
+ * number of seconds since the Epoch.
+ *
+ * TIME_STRING can be in any of the three formats RFC2616 allows the
+ * HTTP servers to emit -- RFC1123-date, RFC850-date or asctime-date,
+ * as well as the time format used in the Set-Cookie header.
+ * Timezones are ignored, and should be GMT.
+ *
+ * Taken from http.c of wget-1.10.2 / Tino Reichardt 2006-09-03
+ * - changed a bit
+ *
+ * return 1 if successful, 0 otherwise
+ */
+#include <time.h>
+extern char *strptime(const char *s, const char *format, struct tm *tm);
+int http_atotm(const char *time_string, struct tm *t)
+{
+ int i;
+
+ static const char *time_formats[] = {
+ "%a, %d %b %Y %T", /* rfc1123: Thu, 29 Jan 1998 22:12:57 */
+ "%A, %d-%b-%y %T", /* rfc850: Thursday, 29-Jan-98 22:12:57 */
+ "%a %b %d %T %Y", /* asctime: Thu Jan 29 22:12:57 1998 */
+ "%a, %d-%b-%Y %T" /* cookies: Thu, 29-Jan-1998 22:12:57 (used
+ in Set-Cookie, defined in the Netscape
+ cookie specification.) */
+ };
+
+ for (i = 0; i < 5; i++)
+ {
+ memset(t, 0, sizeof(struct tm));
+ if (strptime(time_string, time_formats[i], t))
+ return 1;
+ }
+
+ /* All formats have failed. */
+ return 0;
+}
+
+/*
+ * HttpModDate - determine the modification date of a remote file
+ *
+ * return 1 if successful, 0 otherwise
+ */
+GLOBALDEF int HttpModDate(const char *host, const char *path, char *dt, int max,
+ netbuf *nControl)
+{
+ char buf[256];
+ struct tm t;
+ int rv = 0;
+
+ if ((strlen(path) + 7) > sizeof(buf))
+ return 0;
+
+ sprintf(buf, "HEAD %s HTTP/1.1\r\nHost: %s\r\n\r\n", path, host);
+
+ if(!HttpSendCmd(buf,'2',nControl))
+ {
+ if (nControl->response[9] == '3')
+ printf("redirection not supported\n");
+ return 0;
+ }
+
+ while (1)
+ {
+ int ret = 0;
+ char *buf = nControl->response;
+ while (ret < 256) {
+ if (socket_wait(nControl) != 1)
+ return 0;
+ if (net_read(nControl->handle,buf,1) != 1)
+ break;
+ ret++;
+ if (*buf == '\r') continue;
+ if (*buf == '\n') break;
+ buf++;
+ }
+ *buf = 0;
+ if (strstr(nControl->response,"Last-Modified:"))
+ {
+ if (http_atotm(nControl->response+15, &t))
+ {
+ /* 20060830124450 */
+ snprintf(dt, max, "%04d%02d%02d%02d%02d%02d",
+ t.tm_year+1900, t.tm_mon+1, t.tm_mday,
+ t.tm_hour, t.tm_min, t.tm_sec);
+ rv = 1;
+ }
+ }
+ if (strlen(nControl->response) == 0)
+ break;
+ }
+
+ return rv;
+}
+
+/*
* HttpQuit - disconnect from remote
*
* return 1 if successful, 0 otherwise
diff -urN pacman-2.9.8.orig/libftp/ftplib.h pacman-2.9.8/libftp/ftplib.h
--- pacman-2.9.8.orig/libftp/ftplib.h 2004-04-19 09:17:56.000000000 +0200
+++ pacman-2.9.8/libftp/ftplib.h 2006-09-03 17:06:21.000000000 +0200
@@ -120,6 +120,7 @@
GLOBALREF void FtpQuit(netbuf *nControl);
GLOBALREF int HttpConnect(const char *host, unsigned short port, netbuf **nControl);
+GLOBALDEF int HttpModDate(const char *host, const char *path, char *dt, int max, netbuf *nControl);
GLOBALREF int HttpGet(const char *host, const char *output, const char *path,
int *size, netbuf *nControl, unsigned int offset);
GLOBALREF void HttpQuit(netbuf *nControl);
diff -urN pacman-2.9.8.orig/src/pacsync.c pacman-2.9.8/src/pacsync.c
--- pacman-2.9.8.orig/src/pacsync.c 2006-01-31 01:27:43.000000000 +0100
+++ pacman-2.9.8/src/pacsync.c 2006-09-06 19:11:27.000000000 +0200
@@ -1,8 +1,8 @@
/*
* pacsync.c
- *
+ *
* Copyright (c) 2002-2006 by Judd Vinet <jvinet at zeroflux.org>
- *
+ *
* This program is free software; you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation; either version 2 of the License, or
@@ -143,18 +143,17 @@
return(!!downloadfiles_forreal(servers, localpath, files, NULL, NULL));
}
-
/*
* This is the real downloadfiles, used directly by sync_synctree() to check
- * modtimes on remote (ftp only) files.
+ * modtimes on remote (direct ftp/http only) files.
* - if *mtime1 is non-NULL, then only download files
* if they are different than *mtime1. String should be in the form
* "YYYYMMDDHHMMSS" to match the form of ftplib's FtpModDate() function.
* - if *mtime2 is non-NULL, then it will be filled with the mtime
- * of the remote FTP file (from MDTM).
+ * of the remote FTP(MDTM) / HTTP(HEAD Last-Modified) file.
*
- * NOTE: the *mtime option only works for FTP repositories, and won't work
- * if XferCommand is used. We only use it to check mtimes on the
+ * NOTE: the *mtime option only works for FTP/HTTP repositories, and won't
+ * work if XferCommand is used. We only use it to check mtimes on the
* repo db files.
*
* RETURN: 0 for successful download
@@ -344,7 +343,7 @@
filedone = -1;
complete = list_add(complete, fn);
}
- if(mtime2) {
+ if(mtime2) {
strncpy(mtime2, fmtime, 15); /* YYYYMMDDHHMMSS (=14b) */
mtime2[14] = '\0';
}
@@ -372,6 +371,7 @@
char src[PATH_MAX];
char *host;
unsigned port;
+
if(!strcmp(server->protocol, "http") && !pmo_proxyhost) {
/* HTTP servers hang up after each request (but not proxies), so
* we have to re-connect for each file.
@@ -404,13 +404,56 @@
} else {
snprintf(src, PATH_MAX, "%s://%s%s%s", server->protocol, server->server, server->path, fn);
}
- if(!HttpGet(server->server, output, src, &fsz, control, offset)) {
- fprintf(stderr, "\nfailed downloading %s from %s: %s\n",
- src, server->server, FtpLastResponse(control));
- /* we leave the partially downloaded file in place so it can be resumed later */
- } else {
- filedone = 1;
+
+ /* look up Last-Modified Date */
+ if(mtime1 || mtime2) {
+ char fmtime[64];
+ if(!HttpModDate(server->server, src, fmtime, sizeof(fmtime)-1, control)) {
+ fprintf(stderr, "warning: failed to get mtime for %s\n", fn);
+ } else {
+ trim(fmtime);
+ if(mtime1 && !strcmp(mtime1, fmtime)) {
+ /* mtimes are identical, skip this file */
+ vprint("mtimes are identical, skipping %s\n", fn);
+ filedone = -1;
+ complete = list_add(complete, fn);
+ }
+ if(mtime2) {
+ strncpy(mtime2, fmtime, 15); /* YYYYMMDDHHMMSS (=14b) */
+ mtime2[14] = '\0';
+ }
+ }
+ }
+
+ if (!filedone) {
+ host = (pmo_proxyhost) ? pmo_proxyhost : server->server;
+ port = (pmo_proxyhost) ? pmo_proxyport : 80;
+ if(strchr(host, ':')) {
+ vprint("connecting to %s\n", host);
+ } else {
+ vprint("connecting to %s:%u\n", host, port);
+ }
+ /* we have to re-connect */
+ if(!HttpConnect(host, port, &control)) {
+ fprintf(stderr, "error: cannot connect to %s\n", host);
+ continue;
+ }
+ /* set up our progress bar's callback (and idle timeout) */
+ if(strcmp(server->protocol, "file") && control) {
+ FtpOptions(FTPLIB_CALLBACK, (long)log_progress, control);
+ FtpOptions(FTPLIB_IDLETIME, (long)1000, control);
+ FtpOptions(FTPLIB_CALLBACKARG, (long)&fsz, control);
+ FtpOptions(FTPLIB_CALLBACKBYTES, (10*1024), control);
+ }
+ if(!HttpGet(server->server, output, src, &fsz, control, offset)) {
+ fprintf(stderr, "\nfailed downloading %s from %s: %s\n",
+ src, server->server, FtpLastResponse(control));
+ /* we leave the partially downloaded file in place so it can be resumed later */
+ } else {
+ filedone = 1;
+ }
}
+ /* eof http */
} else if(!strcmp(server->protocol, "file")) {
char src[PATH_MAX];
snprintf(src, PATH_MAX, "%s%s", server->path, fn);
-------------- next part --------------
A non-text attachment was scrubbed...
Name: not available
Type: application/pgp-signature
Size: 191 bytes
Desc: not available
URL: <http://archlinux.org/pipermail/pacman-dev/attachments/20060906/622dab03/attachment.pgp>
More information about the pacman-dev
mailing list