Ignore:
Timestamp:
May 7, 2019, 2:17:08 PM (5 years ago)
Author:
djay
Message:

Merge prototype-v0 branch in trunk

Location:
trunk
Files:
2 edited

Legend:

Unmodified
Added
Removed

  • trunk

  • trunk/zoo-project/zoo-kernel/ulinet.c

    r889 r917  
    44 * Author : Gérald FENOY
    55 *
    6  * Copyright (c) 2008-2015 GeoLabs SARL
     6 * Copyright (c) 2008-2019 GeoLabs SARL
    77 *
    88 * Permission is hereby granted, free of charge, to any person obtaining a copy
    … …  
    2929#define MAX_WAIT_MSECS 180*1000 /* Wait max. 180 seconds */
    3030#include "ulinet.h"
     31#include "server_internal.h"
    3132#include <assert.h>
    3233#include <ctype.h>
     34#include "fcgi_stdio.h"
    3335
    3436/**
    … …  
    6971
    7072/**
     73 * Write the downloaded content in the file pouted by the _HINTERNET structure
     74 *
     75 * @param buffer the buffer to read
     76 * @param size size of each member
     77 * @param nmemb number of element to read
     78 * @param data the _HINTERNET structure to write in
     79 * @return the size red, -1 if buffer is NULL
     80 */
     81size_t write_data_into_file(void *buffer, size_t size, size_t nmemb, void *data)
     82{
     83   size_t realsize = size * nmemb;
     84   int writen=0;
     85   _HINTERNET *psInternet;
     86   if(buffer==NULL){
     87     buffer=NULL;
     88     return -1;
     89   }
     90   psInternet=(_HINTERNET *)data;
     91   writen+=fwrite(buffer, size, nmemb, psInternet->file);
     92   fflush(psInternet->file);
     93   psInternet->nDataLen += realsize;
     94
     95   buffer=NULL;
     96   return realsize;
     97}
     98
     99
     100/**
    71101 * In case of presence of "Set-Cookie" in the headers red, store the cookie
    72102 * identifier in cookie
    … …  
    83113    int i;
    84114    char* tmp;
    85     int cnt;
    86     _HINTERNET *psInternet;
    87115    for(i=0;i<12;i++)
    88116#ifndef WIN32
    … …  
    92120#endif
    93121    tmp=strtok((char*) buffer,";"); // knut: added cast to char*
    94     cnt=0;
    95     psInternet=(_HINTERNET *)data;
     122    int cnt=0;
     123    _HINTERNET *psInternet=(_HINTERNET *)data;
    96124    if(tmp!=NULL && psInternet!=NULL){
    97125      psInternet->cookie=(char*)malloc(sizeof(char)*(strlen(tmp)+1));
    … …  
    211239  HINTERNET ret;
    212240  ret.handle=curl_multi_init();
    213   ret.agent=strdup(lpszAgent);
     241  ret.agent=zStrdup(lpszAgent);
    214242  ret.nb=0;
     243  ret.waitingRequests[ret.nb] = NULL;
    215244  ret.ihandle[ret.nb].header=NULL;
     245  ret.ihandle[ret.nb].handle=NULL;
     246  ret.ihandle[ret.nb].hasCacheFile=0;
     247  ret.ihandle[ret.nb].nDataAlloc = 0;
     248  ret.ihandle[ret.nb].url = NULL;
     249  ret.ihandle[ret.nb].mimeType = NULL;
     250  ret.ihandle[ret.nb].cookie = NULL;
     251  ret.ihandle[ret.nb].nDataLen = 0;
     252  ret.ihandle[ret.nb].nDataAlloc = 0;
     253  ret.ihandle[ret.nb].pabyData = NULL;
     254  ret.ihandle[ret.nb].post = NULL;
    216255  return ret;
     256}
     257
     258/**
     259 * Verify if the URL should use a shared cache or not.
     260 *
     261 * In case the security section contains a key named "shared", then if the
     262 * domain listed in the shared key are contained in the url given as parameter
     263 * then it return "SHARED" in other cases, it returns "OTHER".
     264 *
     265 * @param conf the main configuration file maps
     266 * @param url the URL to evaluate
     267 * @return a string "SHARED" in case the host is in a domain listed in the
     268 * shared key, "OTHER" in other cases.
     269 */
     270char* getProvenance(maps* conf,const char* url){
     271  map* sharedCache=getMapFromMaps(conf,"security","shared");
     272  char *res="OTHER";
     273  char *paths[2]={
     274    "mapserverAddress",
     275    "tmpUrl"
     276  };
     277  if(sharedCache!=NULL){
     278    char *hosts=sharedCache->value;
     279    char *curs=strtok(hosts,",");
     280    while(curs!=NULL){
     281      if(strstr(url,curs)==NULL){
     282        return "SHARED";
     283      }
     284      curs=strtok(NULL,",");
     285    }
     286  }
     287  for(int i=0;i<2;i++){
     288    sharedCache=getMapFromMaps(conf,"main",paths[i]);
     289    if(sharedCache!=NULL){
     290      if(strstr(url,sharedCache->value)!=NULL){
     291        return "LOCAL";
     292      }
     293    }
     294  }
     295  return res;
    217296}
    218297
    … …  
    244323 * @return 1 if the host is listed as protected, 0 in other case
    245324 */
    246 int isProtectedHost(const char* protectedHosts,const char* url){
    247   char *token, *saveptr;
    248   int cnt;
    249   char* host;
    250  
    251   // knut: make a copy of url since strtok family modifies first argument and cannot be used on constant strings 
    252   char* urlcpy = (char*) malloc(sizeof(char)*(strlen(url)+1));
    253   urlcpy = strncpy(urlcpy, url, strlen(url)+1); // since count > strlen(url), a null character is properly appended
    254  
    255   //token = strtok_r (url, "//", &saveptr);
    256   token = strtok_r (urlcpy, "//", &saveptr);   // knut
    257   cnt=0;
    258   while(token!=NULL && cnt<=1){
    259     fprintf(stderr,"%s %d %s \n",__FILE__,__LINE__,token);
    260     if(cnt==1)
    261       fprintf(stderr,"%s %d %s \n",__FILE__,__LINE__,strstr(protectedHosts,token));
    262     fflush(stderr);
    263     if(cnt==1 && strstr(protectedHosts,token)!=NULL){
    264       fprintf(stderr,"%s %d %s \n",__FILE__,__LINE__,strstr(protectedHosts,token));
    265       free(urlcpy);
    266       return 1;
    267     }
    268     token = strtok_r (NULL, "/", &saveptr);
    269     cnt+=1;
    270   }
    271   free(urlcpy);
    272   return 0;
     325int isProtectedHost(const char* protectedHosts, const char* url) {
     326        char *token, *saveptr;
     327        int cnt;
     328        char* host;
     329
     330        // knut: make a copy of url since strtok family modifies first argument and cannot be used on constant strings 
     331        char* urlcpy = (char*)malloc(sizeof(char)*(strlen(url) + 1));
     332        urlcpy = strncpy(urlcpy, url, strlen(url) + 1); // since count > strlen(url), a null character is properly appended
     333
     334        //token = strtok_r (url, "//", &saveptr);
     335        token = strtok_r(urlcpy, "//", &saveptr);   // knut
     336        cnt = 0;
     337        while (token != NULL && cnt <= 1) {
     338                fprintf(stderr, "%s %d %s \n", __FILE__, __LINE__, token);
     339                if (cnt == 1)
     340                        fprintf(stderr, "%s %d %s \n", __FILE__, __LINE__, strstr(protectedHosts, token));
     341                fflush(stderr);
     342                if (cnt == 1 && strstr(protectedHosts, token) != NULL) {
     343                        fprintf(stderr, "%s %d %s \n", __FILE__, __LINE__, strstr(protectedHosts, token));
     344                        free(urlcpy);
     345                        return 1;
     346                }
     347                token = strtok_r(NULL, "/", &saveptr);
     348                cnt += 1;
     349        }
     350        free(urlcpy);
     351        return 0;
    273352}
    274353
    … …  
    289368  if(passThrough!=NULL && targetHosts!=NULL){
    290369    char *tmp=zStrdup(passThrough->value);
    291     char *token, *saveptr;
    292     int i;   
    293     token = strtok_r (tmp, ",", &saveptr);
     370    int i;
    294371    for(i=0;i<handle->nb;i++){
    295       if(targetHosts->value[0]=='*' || isProtectedHost(targetHosts->value,handle->ihandle[i].url)==1){
     372      if(strstr(targetHosts->value,"*")!=NULL || isProtectedHost(targetHosts->value,handle->ihandle[i].url)==1){
     373        char *token, *saveptr;
     374        token = strtok_r (tmp, ",", &saveptr);
    296375        while (token != NULL){
    297           int j;
    298       map* tmpMap;       
    299376          int length=strlen(token)+6;
    300377          char* tmp1=(char*)malloc(length*sizeof(char));
     378          map* tmpMap;
    301379          snprintf(tmp1,6,"HTTP_");
     380          int j;
    302381          for(j=0;token[j]!='\0';j++){
    303382            if(token[j]!='-')
    304               tmp1[5+j]=toupper(token[i]);
     383              tmp1[5+j]=toupper(token[j]);
    305384            else
    306385              tmp1[5+j]='_';
    307386            tmp1[5+j+1]='\0';
    308387          }
    309           fprintf(stderr,"%s %d %s \n",__FILE__,__LINE__,tmp1);
    310388          tmpMap = getMapFromMaps(conf,"renv",tmp1);
    311           if(tmpMap!=NULL)         
     389          if(tmpMap!=NULL){
    312390            AddMissingHeaderEntry(&handle->ihandle[i],token,tmpMap->value);
    313           free(tmp1);
     391          }
    314392          if(handle->ihandle[i].header!=NULL)
    315393            curl_easy_setopt(handle->ihandle[i].handle,CURLOPT_HTTPHEADER,handle->ihandle[i].header);
     394          free(tmp1);
    316395          cnt+=1;
    317396          token = strtok_r (NULL, ",", &saveptr);
    … …  
    330409void InternetCloseHandle(HINTERNET* handle0){
    331410  int i=0;
    332   for(i=0;i<handle0->nb;i++){
    333     _HINTERNET handle=handle0->ihandle[i];
    334     if(handle.hasCacheFile>0){
    335       fclose(handle.file);
    336       unlink(handle.filename);
    337     }
    338     else{
    339       handle.pabyData = NULL;
    340       handle.nDataAlloc = handle.nDataLen = 0;
    341     }
    342     if(handle0->ihandle[i].header!=NULL){
    343       curl_slist_free_all(handle0->ihandle[i].header);
    344       handle0->ihandle[i].header=NULL;
    345     }
    346     if(handle.post!=NULL)
    347       free(handle.post);
    348     if(handle.url!=NULL)
    349       free(handle.url);
    350     if(handle.mimeType!=NULL)
    351       free(handle.mimeType);
    352     if(handle.cookie!=NULL)
    353       free(handle.cookie);
    354   }
    355   if(handle0->handle)
    356     curl_multi_cleanup(handle0->handle);
    357   free(handle0->agent);
    358   for(i=handle0->nb-1;i>=0;i--){
    359     free(handle0->waitingRequests[i]);
     411  if(handle0!=NULL){
     412    for(i=0;i<handle0->nb;i++){
     413      _HINTERNET handle=handle0->ihandle[i];
     414      if(handle.hasCacheFile>0){
     415        fclose(handle.file);
     416        unlink(handle.filename);
     417        free(handle.filename);
     418      }
     419      else{
     420        handle.pabyData = NULL;
     421        handle.nDataAlloc = handle.nDataLen = 0;
     422      }
     423      if(handle.header!=NULL){
     424        curl_slist_free_all(handle.header);
     425        handle.header=NULL;
     426      }
     427      if(handle.post!=NULL){
     428        free(handle.post);
     429        handle.post=NULL;
     430      }
     431      if(handle.url!=NULL){
     432        free(handle.url);
     433        handle.url=NULL;
     434      }
     435      if(handle.mimeType!=NULL){
     436        free(handle.mimeType);
     437        handle.mimeType=NULL;
     438      }
     439      if(handle.cookie!=NULL){
     440        free(handle.cookie);
     441        handle.cookie=NULL;
     442      }
     443      if(handle0->waitingRequests[i]!=NULL){
     444        free(handle0->waitingRequests[i]);
     445        handle0->waitingRequests[i]=NULL;
     446      }
     447    }
     448    if(handle0->handle)
     449      curl_multi_cleanup(handle0->handle);
     450    if(handle0->agent!=NULL){
     451      free(handle0->agent);
     452      handle0->agent=NULL;
     453    }
    360454  }
    361455}
    … …  
    370464 * @param dwFlags desired download mode (INTERNET_FLAG_NO_CACHE_WRITE for not using cache file)
    371465 * @param dwContext not used
    372  */
    373 HINTERNET InternetOpenUrl(HINTERNET* hInternet,LPCTSTR lpszUrl,LPCTSTR lpszHeaders,size_t dwHeadersLength,size_t dwFlags,size_t dwContext){
     466 * @param conf the main configuration file maps pointer
     467 * @return the updated HINTERNET
     468 */
     469HINTERNET InternetOpenUrl(HINTERNET* hInternet,LPCTSTR lpszUrl,LPCTSTR lpszHeaders,size_t dwHeadersLength,size_t dwFlags,size_t dwContext,const maps* conf){
    374470
    375471  char filename[255];
     472  int ldwFlags=INTERNET_FLAG_NEED_FILE;
    376473  struct MemoryStruct header;
     474  map* memUse=getMapFromMaps((maps*) conf,"main","memory"); // knut: addad cast to maps*
    377475
    378476  hInternet->ihandle[hInternet->nb].handle=curl_easy_init( );
    … …  
    407505#endif
    408506
    409      
    410   switch(dwFlags)
     507  if(memUse!=NULL && strcasecmp(memUse->value,"load")==0)
     508    ldwFlags=INTERNET_FLAG_NO_CACHE_WRITE;
     509 
     510  switch(ldwFlags)
    411511    {
    412512    case INTERNET_FLAG_NO_CACHE_WRITE:
    413       hInternet->ihandle[hInternet->nb].hasCacheFile=-1;
    414513      curl_easy_setopt(hInternet->ihandle[hInternet->nb].handle, CURLOPT_WRITEFUNCTION, write_data_into);
    415514      curl_easy_setopt(hInternet->ihandle[hInternet->nb].handle, CURLOPT_WRITEDATA, (void*)&hInternet->ihandle[hInternet->nb]);
     515      hInternet->ihandle[hInternet->nb].hasCacheFile=-1;
    416516      break;
    417517    default:
    418       sprintf(hInternet->ihandle[hInternet->nb].filename,"/tmp/ZOO_Cache%d",(int)time(NULL));
    419       hInternet->ihandle[hInternet->nb].filename[24]=0;
    420 #ifdef MSG_LAF_VERBOSE
    421       fprintf(stderr,"file=%s",hInternet->ihandle[hInternet->nb].filename);
    422 #endif
    423       hInternet->ihandle[hInternet->nb].filename=filename;
     518      memset(filename,0,255);
     519      char* tmpUuid=get_uuid();
     520      map* tmpPath=NULL;
     521      if(conf!=NULL){
     522        tmpPath=getMapFromMaps((maps*) conf,"main","tmpPath"); // knut added cast to maps*
     523      }
     524      if(tmpPath==NULL)
     525        sprintf(filename,"/tmp/ZOO_Cache%s", tmpUuid);
     526      else
     527        sprintf(filename,"%s/ZOO_Cache%s", tmpPath->value,tmpUuid);
     528      free(tmpUuid);
     529      hInternet->ihandle[hInternet->nb].filename=zStrdup(filename);
    424530      hInternet->ihandle[hInternet->nb].file=fopen(hInternet->ihandle[hInternet->nb].filename,"w+");
    425    
    426531      hInternet->ihandle[hInternet->nb].hasCacheFile=1;
    427       curl_easy_setopt(hInternet->ihandle[hInternet->nb].handle, CURLOPT_WRITEFUNCTION, NULL);
    428       curl_easy_setopt(hInternet->ihandle[hInternet->nb].handle, CURLOPT_WRITEDATA, hInternet->ihandle[hInternet->nb].file);
    429       hInternet->ihandle[hInternet->nb].nDataLen=0;
     532      curl_easy_setopt(hInternet->ihandle[hInternet->nb].handle, CURLOPT_WRITEFUNCTION, write_data_into_file);
     533      curl_easy_setopt(hInternet->ihandle[hInternet->nb].handle, CURLOPT_WRITEDATA, (void*)&hInternet->ihandle[hInternet->nb]);
    430534      break;
    431535    }
    … …  
    443547    curl_easy_setopt(hInternet->ihandle[hInternet->nb].handle,CURLOPT_VERBOSE,1);
    444548#endif
    445     hInternet->ihandle[hInternet->nb].post=strdup(lpszHeaders);
     549    hInternet->ihandle[hInternet->nb].post=zStrdup(lpszHeaders);
    446550    curl_easy_setopt(hInternet->ihandle[hInternet->nb].handle,CURLOPT_POSTFIELDS,hInternet->ihandle[hInternet->nb].post);
    447551    curl_easy_setopt(hInternet->ihandle[hInternet->nb].handle,CURLOPT_POSTFIELDSIZE,(long)dwHeadersLength);
    … …  
    457561  hInternet->ihandle[hInternet->nb].header=NULL;
    458562  ++hInternet->nb;
     563  hInternet->ihandle[hInternet->nb].header=NULL;
    459564
    460565#ifdef ULINET_DEBUG
    … …  
    472577 */
    473578int processDownloads(HINTERNET* hInternet){
    474   int still_running=0;
     579  int still_running=0,numfds;
    475580  int msgs_left=0;
    476581  int i=0;
    477582  do{
    478     curl_multi_perform(hInternet->handle, &still_running);
     583    CURLMcode mc;
     584    mc = curl_multi_perform(hInternet->handle, &still_running);
     585    if(mc==CURLM_OK){
     586#if LIBCURL_VERSION_MINOR >= 28
     587      mc = curl_multi_wait(hInternet->handle, NULL, 0, 1000, &numfds);
     588#else
     589      struct timeval timeout;
     590      fd_set fdread;
     591      fd_set fdwrite;
     592      fd_set fdexcep;
     593      int maxfd = -1;
     594
     595      long curl_timeo = -1;
     596
     597      FD_ZERO(&fdread);
     598      FD_ZERO(&fdwrite);
     599      FD_ZERO(&fdexcep);
     600
     601      /* set a suitable timeout to play around with */
     602      timeout.tv_sec = 1;
     603      timeout.tv_usec = 0;
     604
     605      curl_multi_timeout(hInternet->handle, &curl_timeo);
     606      if(curl_timeo >= 0) {
     607        timeout.tv_sec = curl_timeo / 1000;
     608        if(timeout.tv_sec > 1)
     609          timeout.tv_sec = 1;
     610        else
     611          timeout.tv_usec = (curl_timeo % 1000) * 1000;
     612      }
     613
     614      /* get file descriptors from the transfers */
     615      mc = curl_multi_fdset(hInternet->handle, &fdread, &fdwrite, &fdexcep, &maxfd);
     616#endif
     617    }
     618    if(mc != CURLM_OK) {
     619      fprintf(stderr, "curl_multi failed, code %d.n", mc);
     620      break;
     621    }
    479622  }while(still_running); 
    480623  for(i=0;i<hInternet->nb;i++){
    … …  
    482625    curl_easy_getinfo(hInternet->ihandle[i].handle,CURLINFO_CONTENT_TYPE,&tmp);
    483626    if(tmp!=NULL)
    484       hInternet->ihandle[i].mimeType=strdup(tmp);
     627      hInternet->ihandle[i].mimeType=zStrdup(tmp);
    485628    curl_easy_getinfo(hInternet->ihandle[i].handle,CURLINFO_RESPONSE_CODE,&hInternet->ihandle[i].code);
    486629    curl_multi_remove_handle(hInternet->handle, hInternet->ihandle[i].handle);
    … …  
    498641 */
    499642int freeCookieList(HINTERNET hInternet){
    500   if(hInternet.ihandle[hInternet.nb].cookie)
    501     free(hInternet.ihandle[hInternet.nb].cookie);
    502   hInternet.ihandle[hInternet.nb].cookie=NULL;
     643  hInternet.ihandle[hInternet.nb].cookie=zStrdup("");
    503644#ifndef TIGER
    504645  curl_easy_setopt(hInternet.ihandle[hInternet.nb].handle, CURLOPT_COOKIELIST, "ALL");
    … …  
    522663    fseek (hInternet.file , 0 , SEEK_END);
    523664    dwDataSize=ftell(hInternet.file); //taille du ficher
    524     rewind (hInternet.file);
     665    //dwDataSize=hInternet.nDataLen;
     666    //rewind (hInternet.file);
     667    fseek(hInternet.file, 0, SEEK_SET);
    525668  }
    526669  else{
    … …  
    532675  }
    533676
    534   if( dwNumberOfBytesToRead /* buffer size */ < dwDataSize )
     677  if( dwNumberOfBytesToRead /* buffer size */ < dwDataSize ){
    535678    return 0;
     679  }
    536680
    537681#ifdef MSG_LAF_VERBOSE
    538   printf("\nfile size : %dko\n",dwDataSize/1024);
     682  fprintf(stderr,"\nfile size : %dko\n",dwDataSize/1024);
    539683#endif
    540684
    541685  if(hInternet.hasCacheFile>0){
    542     *lpdwNumberOfBytesRead = fread(lpBuffer,1,dwDataSize,hInternet.file);
     686    int freadRes = fread(lpBuffer,dwDataSize+1,1,hInternet.file);
     687    *lpdwNumberOfBytesRead = hInternet.nDataLen;
    543688  }
    544689  else{
Note: See TracChangeset for help on using the changeset viewer.

Download now !

Search

Context Navigation

Main Navigation

ZOO Sponsors

http://www.zoo-project.org/trac/chrome/site/img/geolabs-logo.pnghttp://www.zoo-project.org/trac/chrome/site/img/neogeo-logo.png http://www.zoo-project.org/trac/chrome/site/img/apptech-logo.png http://www.zoo-project.org/trac/chrome/site/img/3liz-logo.png http://www.zoo-project.org/trac/chrome/site/img/gateway-logo.png

Become a sponsor !

Knowledge partners

http://www.zoo-project.org/trac/chrome/site/img/ocu-logo.png http://www.zoo-project.org/trac/chrome/site/img/gucas-logo.png http://www.zoo-project.org/trac/chrome/site/img/polimi-logo.png http://www.zoo-project.org/trac/chrome/site/img/fem-logo.png http://www.zoo-project.org/trac/chrome/site/img/supsi-logo.png http://www.zoo-project.org/trac/chrome/site/img/cumtb-logo.png

Become a knowledge partner

Related links

http://zoo-project.org/img/ogclogo.png http://zoo-project.org/img/osgeologo.png