source: branches/prototype-v0/zoo-project/zoo-kernel/caching.c @ 822

Last change on this file since 822 was 822, checked in by djay, 3 years ago

Commit the minimal requirements for remote HPC support

  • Property svn:keywords set to Id
File size: 13.4 KB
Line 
1/*
2 * Author : Gérald Fenoy
3 *
4 *  Copyright 2008-2015 GeoLabs SARL. All rights reserved.
5 *
6 * Permission is hereby granted, free of charge, to any person obtaining a copy
7 * of this software and associated documentation files (the "Software"), to deal
8 * in the Software without restriction, including without limitation the rights
9 * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
10 * copies of the Software, and to permit persons to whom the Software is
11 * furnished to do so, subject to the following conditions:
12 *
13 * The above copyright notice and this permission notice shall be included in
14 * all copies or substantial portions of the Software.
15 *
16 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
17 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
18 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
19 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
20 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
21 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
22 * THE SOFTWARE.
23 */
24
25#include "caching.h"
26#include "service.h"
27#include "service_internal.h"
28#include "response_print.h"
29#include <openssl/md5.h>
30#include <openssl/hmac.h>
31#include <openssl/evp.h>
32#include <openssl/bio.h>
33#include <openssl/buffer.h>
34
35/**
36 * Compute md5
37 *
38 * @param url the char*
39 * @return a char* representing the md5 of the url
40 * @warning make sure to free resources returned by this function
41 */
42char* getMd5(char* url){
43  EVP_MD_CTX md5ctx;
44  char* fresult=(char*)malloc((EVP_MAX_MD_SIZE+1)*sizeof(char));
45  unsigned char result[EVP_MAX_MD_SIZE];
46  unsigned int len;
47  EVP_DigestInit(&md5ctx, EVP_md5());
48  EVP_DigestUpdate(&md5ctx, url, strlen(url));
49  EVP_DigestFinal_ex(&md5ctx,result,&len);
50  EVP_MD_CTX_cleanup(&md5ctx);
51  int i;
52  for(i = 0; i < len; i++){
53    if(i>0){
54      char *tmp=strdup(fresult);
55      sprintf(fresult,"%s%02x", tmp,result[i]);
56      free(tmp);
57    }
58    else
59      sprintf(fresult,"%02x",result[i]);
60  }
61  return fresult;
62}
63
64
65/**
66 * Cache a file for a given request.
67 * For each cached file, the are two files stored, a .zca and a .zcm containing
68 * the downloaded content and the mimeType respectively.
69 *
70 * @param conf the maps containing the settings of the main.cfg file
71 * @param request the url used too fetch the content
72 * @param content the downloaded content
73 * @param mimeType the content mimeType
74 * @param length the content size
75 * @param filepath a buffer for storing the path of the cached file; may be NULL
76 * @param max_path the size of the allocated filepath buffer 
77 */
78void addToCache(maps* conf,char* request,char* content,char* mimeType,int length, 
79                char* filepath, size_t max_path){
80  map* tmp=getMapFromMaps(conf,"main","cacheDir");
81  if(tmp!=NULL){
82    char* md5str=getMd5(request);
83    char* fname=(char*)malloc(sizeof(char)*(strlen(tmp->value)+strlen(md5str)+6));
84    sprintf(fname,"%s/%s.zca",tmp->value,md5str);
85#ifdef DEBUG
86    fprintf(stderr,"Cache list : %s\n",fname);
87    fflush(stderr);
88#endif
89    FILE* fo=fopen(fname,"w+");
90    if(fo==NULL){
91#ifdef DEBUG
92      fprintf (stderr, "Failed to open %s for writing: %s\n",fname, strerror(errno));
93#endif
94      filepath = NULL; 
95      return;
96    }
97    fwrite(content,sizeof(char),length,fo);
98    fclose(fo);
99       
100        if (filepath != NULL) {
101                strncpy(filepath, fname, max_path);
102        }       
103
104    sprintf(fname,"%s/%s.zcm",tmp->value,md5str);
105    fo=fopen(fname,"w+");
106#ifdef DEBUG
107    fprintf(stderr,"MIMETYPE: %s\n",mimeType);
108#endif
109    fwrite(mimeType,sizeof(char),strlen(mimeType),fo);
110    fclose(fo);
111
112    free(md5str);
113    free(fname);
114  }
115  else {
116          filepath = NULL;
117  }       
118}
119
120/**
121 * Verify if a url is available in the cache
122 *
123 * @param conf the maps containing the settings of the main.cfg file
124 * @param request the url
125 * @return the full name of the cached file if any, NULL in other case
126 * @warning make sure to free resources returned by this function (if not NULL)
127 */
128char* isInCache(maps* conf,char* request){
129  map* tmpM=getMapFromMaps(conf,"main","cacheDir");
130  if(tmpM!=NULL){
131    char* md5str=getMd5(request);
132#ifdef DEBUG
133    fprintf(stderr,"MD5STR : (%s)\n\n",md5str);
134#endif
135    char* fname=(char*)malloc(sizeof(char)*(strlen(tmpM->value)+strlen(md5str)+6));
136    sprintf(fname,"%s/%s.zca",tmpM->value,md5str);
137    struct stat f_status;
138    int s=stat(fname, &f_status);
139    if(s==0 && f_status.st_size>0){
140      free(md5str);
141      return fname;
142    }
143    free(md5str);
144    free(fname);
145  }
146  return NULL;
147}
148
149/**
150 * Read the downloaded file for a specific input
151 *
152 * @param m the maps containing the settings of the main.cfg file
153 * @param in the input
154 * @param index the input index
155 * @param hInternet the internet connection
156 * @param error the error map pointer
157 * @return 0 in case of success, -1 in case of failure
158 */
159int readCurrentInput(maps** m,maps** in,int* index,HINTERNET* hInternet,map** error){
160  map* tmp1;
161  char sindex[5];
162  maps* content=*in;
163  map* length=getMap(content->content,"length");
164  int shouldClean=-1;
165  if(length==NULL){
166    length=createMap("length","1");
167    shouldClean=1;
168  }
169  for(int i=0;i<atoi(length->value);i++){
170    char* fcontent;
171    char *mimeType=NULL;
172    int fsize=0;
173    char cname[15];
174    char vname[11];
175    char vname1[11];
176    char sname[9];
177    char mname[15];
178    char icname[14];
179    char xname[16];
180    char bname[8];
181    char hname[11];
182    char oname[12];
183    if(*index>0)
184      sprintf(vname1,"value_%d",*index);
185    else
186      sprintf(vname1,"value");
187   
188    if(i>0){
189      tmp1=getMap(content->content,cname);
190      sprintf(cname,"cache_file_%d",i);
191      sprintf(vname,"value_%d",i);
192      sprintf(sname,"size_%d",i);
193      sprintf(mname,"mimeType_%d",i);
194      sprintf(icname,"isCached_%d",i);
195      sprintf(xname,"Reference_%d",i);
196      sprintf(bname,"body_%d",i);
197      sprintf(hname,"headers_%d",i);
198      sprintf(oname,"Order_%d",i);
199    }else{
200      sprintf(cname,"cache_file");
201      sprintf(vname,"value");
202      sprintf(sname,"size");
203      sprintf(mname,"mimeType");
204      sprintf(icname,"isCached");
205      sprintf(xname,"Reference");
206      sprintf(bname,"body");
207      sprintf(hname,"headers",i);
208      sprintf(oname,"Order");
209    }
210   
211    map* tmap=getMap(content->content,oname);
212    sprintf(sindex,"%d",*index+1);
213    if((tmp1=getMap(content->content,xname))!=NULL && tmap!=NULL && strcasecmp(tmap->value,sindex)==0){
214     
215      if(getMap(content->content,icname)==NULL){
216        fcontent=(char*)malloc((hInternet->ihandle[*index].nDataLen+1)*sizeof(char));
217        if(fcontent == NULL){
218          errorException(*m, _("Unable to allocate memory"), "InternalError",NULL);
219          return -1;
220        }
221        size_t dwRead;
222        InternetReadFile(hInternet->ihandle[*index], 
223                         (LPVOID)fcontent, 
224                         hInternet->ihandle[*index].nDataLen, 
225                         &dwRead);
226        fcontent[hInternet->ihandle[*index].nDataLen]=0;
227        fsize=hInternet->ihandle[*index].nDataLen;
228        if(hInternet->ihandle[*index].mimeType==NULL)
229          mimeType=zStrdup("none");
230        else
231          mimeType=zStrdup(hInternet->ihandle[*index].mimeType);             
232       
233        map* tmpMap=getMapOrFill(&(*in)->content,vname,"");
234        free(tmpMap->value);
235        tmpMap->value=(char*)malloc((fsize+1)*sizeof(char));
236        if(tmpMap->value==NULL){
237          return errorException(*m, _("Unable to allocate memory"), "InternalError",NULL);
238        }
239        memcpy(tmpMap->value,fcontent,(fsize+1)*sizeof(char));
240        if(hInternet->ihandle[*index].code!=200){
241          char *error_rep_str=_("Unable to download the file for the input <%s>, response code was : %d.");
242          char *error_msg=(char*)malloc((strlen(error_rep_str)+strlen(content->name)+4)*sizeof(char));
243          sprintf(error_msg,error_rep_str,content->name,hInternet->ihandle[*index].code);
244          if(*error==NULL){
245            *error=createMap("text",error_msg);
246            addToMap(*error,"locator",content->name);
247            addToMap(*error,"code","InvalidParameterValue");
248          }else{
249            int nb=1;
250            map* tmpMap=getMap(*error,"length");
251            if(tmpMap!=NULL)
252              nb=atoi(tmpMap->value);
253            setMapArray(*error,"text",nb,error_msg);
254            setMapArray(*error,"locator",nb,content->name);
255            setMapArray(*error,"code",nb,"InvalidParameterValue");
256          }
257          return -1;
258        }
259       
260        char ltmp1[256];
261        sprintf(ltmp1,"%d",fsize);
262        map* tmp=getMapFromMaps(*m,"main","cacheDir");
263        char *request=NULL;
264        if(tmp!=NULL){
265          map* tmp2;
266          char* md5str=NULL;
267          if((tmp2=getMap(content->content,bname))!=NULL){
268            dumpMap(tmp2);
269            char *tmpStr=(char*)malloc((strlen(tmp1->value)+strlen(tmp2->value)+1)*sizeof(char));
270            sprintf(tmpStr,"%s%s",tmp1->value,tmp2->value);
271            if((tmp2=getMap(content->content,"headers"))!=NULL){
272              char *tmpStr2=zStrdup(tmpStr);
273              tmpStr=(char*)malloc((strlen(tmpStr2)+strlen(tmp2->value)+1)*sizeof(char));
274              sprintf(tmpStr,"%s%s",tmpStr2,tmp2->value);
275              free(tmpStr2);
276            }
277            md5str=getMd5(tmpStr);
278            request=zStrdup(tmpStr);
279            free(tmpStr);
280          }else{
281            md5str=getMd5(tmp1->value);
282            request=zStrdup(tmp1->value);
283          }
284          char* fname=(char*)malloc(sizeof(char)*(strlen(tmp->value)+strlen(md5str)+6));
285          sprintf(fname,"%s/%s.zca",tmp->value,md5str);
286          addToMap((*in)->content,cname,fname);
287          free(fname);
288        }
289        addToMap((*in)->content,sname,ltmp1);
290        addToMap((*in)->content,mname,mimeType);
291        addToCache(*m,request,fcontent,mimeType,fsize, NULL, 0);
292        free(fcontent);
293        free(mimeType);
294        free(request);
295        *index++;
296       
297      }
298    }
299  }
300  if(shouldClean>0){
301    freeMap(&length);
302    free(length);
303  }
304  return 0;
305}
306
307/**
308 * Effectively run all the HTTP requests in the queue
309 *
310 * @param m the maps containing the settings of the main.cfg file
311 * @param inputs the maps containing the inputs (defined in the requests+added
312 *  per default based on the zcfg file)
313 * @param hInternet the HINTERNET pointer
314 * @param error the error map pointer
315 * @return 0 on success, -1 on failure
316 */
317int runHttpRequests(maps** m,maps** inputs,HINTERNET* hInternet,map** error){
318  int hasAFailure=0;
319  if(hInternet!=NULL && hInternet->nb>0){
320    AddHeaderEntries(hInternet,*m);
321    processDownloads(hInternet);
322    maps* content=*inputs;
323    int index=0;
324    while(content!=NULL){
325      if(content->child!=NULL){
326        maps* cursor=content->child;
327        while(cursor!=NULL){
328          int red=readCurrentInput(m,&cursor,&index,hInternet,error);
329          if(red<0)
330            hasAFailure=red;
331          cursor=cursor->next;
332        }
333      }
334      else{
335        int red=readCurrentInput(m,&content,&index,hInternet,error);
336        if(red<0)
337          hasAFailure=red;
338      }
339      content=content->next;
340    }
341  }
342  return hasAFailure;
343}
344
345/**
346 * Add a request in the download queue
347 *
348 * @param m the maps containing the settings of the main.cfg file
349 * @param url the url to add to the queue
350 */
351void addRequestToQueue(maps** m,HINTERNET* hInternet,const char* url,bool req){
352  hInternet->waitingRequests[hInternet->nb]=strdup(url);
353  if(req)
354    InternetOpenUrl(hInternet,hInternet->waitingRequests[hInternet->nb],NULL,0,INTERNET_FLAG_NO_CACHE_WRITE,0);
355  maps *oreq=getMaps(*m,"orequests");
356  if(oreq==NULL){
357    oreq=createMaps("orequests");
358    oreq->content=createMap("value",url);
359    addMapsToMaps(m,oreq);
360    freeMaps(&oreq);
361    free(oreq);
362  }else{
363    setMapArray(oreq->content,"value",hInternet->nb-1,url);
364  }
365}
366
367/**
368 * Try to load file from cache or download a remote file if not in cache
369 *
370 * @param m the maps containing the settings of the main.cfg file
371 * @param content the map to update
372 * @param hInternet the HINTERNET pointer
373 * @param url the url to fetch
374 * @return 0
375 */
376int loadRemoteFile(maps** m,map** content,HINTERNET* hInternet,char *url){
377  char* fcontent = NULL;
378  char* cached=isInCache(*m,url);
379  char *mimeType=NULL;
380  int fsize=0;
381
382  map* t=getMap(*content,"xlink:href");
383  if(t==NULL){
384    t=getMap((*content),"href");
385    addToMap(*content,"xlink:href",url);
386  }
387
388  if(cached!=NULL){
389
390    struct stat f_status;
391    int s=stat(cached, &f_status);
392    if(s==0){
393      fcontent=(char*)malloc(sizeof(char)*(f_status.st_size+1));
394      FILE* f=fopen(cached,"rb");
395      fread(fcontent,f_status.st_size,1,f);
396      fsize=f_status.st_size;
397      fcontent[fsize]=0;
398      fclose(f);
399      addToMap(*content,"cache_file",cached);
400    }
401    cached[strlen(cached)-1]='m';
402    s=stat(cached, &f_status);
403    if(s==0){
404      mimeType=(char*)malloc(sizeof(char)*(f_status.st_size+1));
405      FILE* f=fopen(cached,"rb");
406      fread(mimeType,f_status.st_size,1,f);
407      mimeType[f_status.st_size]=0;
408      fclose(f);
409    }
410
411  }else{   
412    addRequestToQueue(m,hInternet,url,true);
413    return 0;
414  }
415  if(fsize==0){
416    return errorException(*m, _("Unable to download the file."), "InternalError",NULL);
417  }
418  if(mimeType!=NULL){
419    addToMap(*content,"fmimeType",mimeType);
420  }
421
422  map* tmpMap=getMapOrFill(content,"value","");
423   
424  free(tmpMap->value);
425  tmpMap->value=(char*)malloc((fsize+1)*sizeof(char));
426  if(tmpMap->value==NULL || fcontent == NULL)
427    return errorException(*m, _("Unable to allocate memory"), "InternalError",NULL);
428  memcpy(tmpMap->value,fcontent,(fsize+1)*sizeof(char));
429
430  char ltmp1[256];
431  sprintf(ltmp1,"%d",fsize);
432  addToMap(*content,"size",ltmp1);
433  if(cached==NULL){
434    addToCache(*m,url,fcontent,mimeType,fsize, NULL, 0);
435  }
436  else{
437    addToMap(*content,"isCached","true");
438    map* tmp=getMapFromMaps(*m,"main","cacheDir");
439    if(tmp!=NULL){
440      map *c=getMap((*content),"xlink:href");
441      char* md5str=getMd5(c->value);
442      char* fname=(char*)malloc(sizeof(char)*(strlen(tmp->value)+strlen(md5str)+6));
443      sprintf(fname,"%s/%s.zca",tmp->value,md5str);
444      addToMap(*content,"cache_file",fname);
445      free(fname);
446    }
447  }
448  free(fcontent);
449  free(mimeType);
450  free(cached);
451  return 0;
452}
Note: See TracBrowser for help on using the repository browser.

Search

ZOO Sponsors

http://www.zoo-project.org/trac/chrome/site/img/geolabs-logo.pnghttp://www.zoo-project.org/trac/chrome/site/img/neogeo-logo.png http://www.zoo-project.org/trac/chrome/site/img/apptech-logo.png http://www.zoo-project.org/trac/chrome/site/img/3liz-logo.png http://www.zoo-project.org/trac/chrome/site/img/gateway-logo.png

Become a sponsor !

Knowledge partners

http://www.zoo-project.org/trac/chrome/site/img/ocu-logo.png http://www.zoo-project.org/trac/chrome/site/img/gucas-logo.png http://www.zoo-project.org/trac/chrome/site/img/polimi-logo.png http://www.zoo-project.org/trac/chrome/site/img/fem-logo.png http://www.zoo-project.org/trac/chrome/site/img/supsi-logo.png http://www.zoo-project.org/trac/chrome/site/img/cumtb-logo.png

Become a knowledge partner

Related links

http://zoo-project.org/img/ogclogo.png http://zoo-project.org/img/osgeologo.png