Cache/proxy documentation

I found the use of caches and proxies in the library poorly documented on some 
aspects.

My first problem was to declare a proxy and have a user identifier and
password automatically sent to the proxy instead of being asked to the user
at each request. After browsing the code and the documentation about caches
and proxies, the only solution I found was to directly call the function 
HTAA_updateNode() from my program, and to do that I had to duplicate the
structure HTBasic.
    typedef struct { char *uid; char *pw; BOOL retry; BOOL proxy; } myHTBasic;
    myHTBasic *basic = (myHTBasic *) calloc(1, sizeof(myHTBasic));
    basic->uid  = strdup(PROXY_UID);
    basic->pw = my_strdup(PROXY_PASSWD);
    basic->retry= NO;  basic->proxy = YES;
    HTAA_updateNode(YES,"basic",PROXY_REALM,HTTP_PROXY,basic);
The documentation didn't help me to find this solution.


My second problem was that files given back by the proxy were sometimes not
the latest files. The documentation on proxies or cache management did not
lead me to a solution. Only recently a network specialist found one for me:
    HTRequest_addGnHd(W3C_Request,HT_G_PRAGMA_NO_CACHE);
This command and pragma is refered in HTReq.html#gnhd (section
"HTTP Header Mask"). The use of the pragma is not explained so I could
not retrieve it with a "grep cache *.html" or "grep proxy *.html".
There is also no reference to it in other sections.


The libwww is flexible but complex. I think that the following program 
would be helpful if given in the list of examples. It's called wwwcp.
It copy a WWW-accessible file (parameter 1) into a local file (parameter 2).
I do not know if the solution I give is optimal or fully correct (it just
work correctly when I use it). Please tell me if it is not.


Regards,

Philippe


/**************************** wwwcp.c **************************************
 Copy a WWW-accessible file (arg. 1: URL) into a local file (arg. 2)

 Compilation without proxy:
gcc -DHTTP_PROXY='""' -DFTP_PROXY='""' -DNEWS_PROXY='""'   \
    -DWAIS_PROXY='""' -DGOPHER_PROXY='""'                  \
    -DNO_PROXY='""'   -DNO_PROXY_PORT=0 -DPROXY_REALM='""' \
    -DPROXY_UID='""' -DPROXY_PASSWD='""'                   \
    -I./w3c-libwww-5.1l/lib/include/w3c-libwww             \
    -DHAVE_CONFIG_H -Wall -I. \
    -o wwwcp.o -c wwwcp.c
gcc -o wwwcp -g -Wall  wwwcp.o  ./w3c-libwww-5.1l/lib/lib/libwww.a

 Compilation with a proxy (here with Griffith Uni proxy):
gcc -DHTTP_PROXY='"http://webcache.gu.edu.au:8080"'  \
    -DFTP_PROXY='"ftp://webcache.gu.edu.au:8080"'    \
    -DNEWS_PROXY='"news://webcache.gu.edu.au:8080"'  \
    -DWAIS_PROXY='"wais://webcache.gu.edu.au:8080"'  \
    -DGOPHER_PROXY='"gother://webcache.gu.edu.au:8080"'  \
    -DNO_PROXY='"www.gu.edu.au"' -DNO_PROXY_PORT=0       \
    -DPROXY_REALM='"Griffith University Internet Login"' \
    -DPROXY_UID='"an_uid"' -DPROXY_PASSWD='"a_passwd"'   \
    -I./w3c-libwww-5.1l/lib/include/w3c-libwww           \
    -DHAVE_CONFIG_H -Wall -I. \
    -o wwwcp.o -c wwwcp.c
gcc -o wwwcp -g -Wall  wwwcp.o  ./w3c-libwww-5.1l/lib/lib/libwww.a
****************************************************************************/

#include "WWWLib.h"
#include "WWWHTTP.h"
#include "WWWInit.h"

BOOL loadURLtoFile (char *absolute_url,char *fileName);
void W3CinitForURLRequests();
void W3CendForURLRequests();

HTRequest *W3C_Request=NULL; 
HTList *ResultList=NULL;
static char *ThisProgramName;


int main (int argc, char *argv[])
{ int status; char tmpFileName[L_tmpnam]; tmpnam(tmpFileName);
  if (!(ThisProgramName=strrchr(argv[0],'/'))) ThisProgramName=argv[0];

  if (argc!=3) { printf("Usage: %s URL localFileName\n",argv[0]); return 1; }
  W3CinitForURLRequests();
  status= !loadURLtoFile(argv[1],argv[2]);
  W3CendForURLRequests();
  return status;
}


BOOL loadURLtoFile (char *absolute_url, char *fileName)
{ if (!absolute_url || !*absolute_url)
  { fprintf(stderr,"\nNo URL to load\n"); return NO; }
  else
  { struct stat fileStat;   BOOL ok=NO;
    if (!absolute_url)
    {  fprintf(stderr,"\n%s: out of memory - Cannot load %s to %s\n",
                      ThisProgramName, absolute_url, fileName);
      return NO;
    }
    if (!(ok= HTLoadToFile(absolute_url,W3C_Request,fileName)))
      fprintf(stderr,"\n%s: cannot load %s to %s via %s. Check the "
                     " spelling or the access rights\n",
                     ThisProgramName, absolute_url, fileName, HTTP_PROXY);
    else
      if ((stat(fileName, &fileStat)==-1) || (fileStat.st_size==0))
      { ok=NO; fprintf(stderr,"\n%s: could not load %s to %s via %s. "
                       "Check spelling or access rights\n",
                       ThisProgramName, absolute_url, fileName, HTTP_PROXY);
      }
      else fprintf(stderr,"\n%s: %s loaded to %s via %s\n",
                          ThisProgramName,absolute_url,fileName,HTTP_PROXY);

    chmod(fileName,S_IRWXU | S_IRWXG | S_IRWXO);
    return ok;
  }
}




typedef struct { char *uid; char *pw; BOOL retry; BOOL proxy; } myHTBasic;
/* copy of the local type "HTBasic" in $(W3C)/Library/src/HTAABrow.c */


void  W3CinitForURLRequests()
{ HTProfile_newPreemptiveClient("IR_App", "1.0");
#if 1
  if (HTTP_PROXY[0] && !HTProxy_add("http",HTTP_PROXY))
     fprintf(stderr,"\nThe proxy %s cannot be set\n",HTTP_PROXY);
  if (FTP_PROXY[0] && !HTProxy_add("ftp",FTP_PROXY))
     fprintf(stderr,"\nThe proxy %s cannot be set\n",FTP_PROXY);
  if (NEWS_PROXY[0] && !HTProxy_add("news",NEWS_PROXY))
     fprintf(stderr,"\nThe proxy %s cannot be set\n",NEWS_PROXY);
  if (WAIS_PROXY[0] && !HTProxy_add("wais",WAIS_PROXY))
     fprintf(stderr,"\nThe proxy %s cannot be set\n",WAIS_PROXY);
  if (GOPHER_PROXY[0] && !HTProxy_add("gother",GOPHER_PROXY))
     fprintf(stderr,"\nThe proxy %s cannot be set\n",GOPHER_PROXY);
  if (NO_PROXY[0] && !HTNoProxy_add(NO_PROXY,NULL,NO_PROXY_PORT))
     fprintf(stderr,"\nCannot invalidate proxy for %s\n",NO_PROXY);
#else
  /* HTProxy_getEnvVar(); //does not work */
#endif
  HTSetTraceMessageMask("h");
  if (PROXY_PASSWD[0])
  { myHTBasic *basic = (myHTBasic *) HT_CALLOC(1, sizeof(myHTBasic));
    basic->uid  = strdup(PROXY_UID);  basic->pw = strdup(PROXY_PASSWD);
    basic->retry= NO;  basic->proxy = YES;
    HTAA_updateNode(YES,"basic",PROXY_REALM,HTTP_PROXY,basic);
  }
  W3C_Request = HTRequest_new(); 
  if (!HTRequest_setProxy(W3C_Request,HTTP_PROXY))
     fprintf(stderr,"\nThe proxy %s cannot be set\n",HTTP_PROXY);
  else fprintf(stderr,"\nThe proxy %s has been set\n",HTTP_PROXY);

  HTRequest_setOutputFormat(W3C_Request, WWW_SOURCE);
  HTRequest_addGnHd(W3C_Request,HT_G_PRAGMA_NO_CACHE);
}


void  W3CendForURLRequests()
{ HTRequest_delete(W3C_Request);  HTProfile_delete();
}

Received on Tuesday, 23 March 1999 00:24:19 UTC