source: trunk/zoo-project/zoo-kernel/caching.c @ 837

Last change on this file since 837 was 837, checked in by jmckenna, 3 years ago

properly initialize constant on Windows (ticket#159)

  • Property svn:keywords set to Id
File size: 12.6 KB
Line 
1/*
2 * Author : Gérald Fenoy
3 *
4 *  Copyright 2008-2015 GeoLabs SARL. All rights reserved.
5 *
6 * Permission is hereby granted, free of charge, to any person obtaining a copy
7 * of this software and associated documentation files (the "Software"), to deal
8 * in the Software without restriction, including without limitation the rights
9 * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
10 * copies of the Software, and to permit persons to whom the Software is
11 * furnished to do so, subject to the following conditions:
12 *
13 * The above copyright notice and this permission notice shall be included in
14 * all copies or substantial portions of the Software.
15 *
16 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
17 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
18 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
19 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
20 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
21 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
22 * THE SOFTWARE.
23 */
24
25#include "caching.h"
26#include "service.h"
27#include "service_internal.h"
28#include "response_print.h"
29#include <openssl/md5.h>
30#include <openssl/hmac.h>
31#include <openssl/evp.h>
32#include <openssl/bio.h>
33#include <openssl/buffer.h>
34
35/**
36 * Compute md5
37 *
38 * @param url the char*
39 * @return a char* representing the md5 of the url
40 * @warning make sure to free resources returned by this function
41 */
42char* getMd5(char* url){
43  EVP_MD_CTX md5ctx;
44  char* fresult=(char*)malloc((EVP_MAX_MD_SIZE+1)*sizeof(char));
45  unsigned char result[EVP_MAX_MD_SIZE];
46  unsigned int len;
47  EVP_DigestInit(&md5ctx, EVP_md5());
48  EVP_DigestUpdate(&md5ctx, url, strlen(url));
49  EVP_DigestFinal_ex(&md5ctx,result,&len);
50  EVP_MD_CTX_cleanup(&md5ctx);
51  int i;
52  for(i = 0; i < len; i++){
53    if(i>0){
54      char *tmp=strdup(fresult);
55      sprintf(fresult,"%s%02x", tmp,result[i]);
56      free(tmp);
57    }
58    else
59      sprintf(fresult,"%02x",result[i]);
60  }
61  return fresult;
62}
63
64
65/**
66 * Cache a file for a given request.
67 * For each cached file, the are two files stored, a .zca and a .zcm containing
68 * the downloaded content and the mimeType respectively.
69 *
70 * @param conf the maps containing the settings of the main.cfg file
71 * @param request the url used too fetch the content
72 * @param content the downloaded content
73 * @param mimeType the content mimeType
74 * @param length the content size
75 * @param filepath a buffer for storing the path of the cached file; may be NULL
76 * @param max_path the size of the allocated filepath buffer 
77 */
78void addToCache(maps* conf,char* request,char* content,char* mimeType,int length, 
79                char* filepath, size_t max_path){
80  map* tmp=getMapFromMaps(conf,"main","cacheDir");
81  if(tmp!=NULL){
82    char* md5str=getMd5(request);
83    char* fname=(char*)malloc(sizeof(char)*(strlen(tmp->value)+strlen(md5str)+6));
84    sprintf(fname,"%s/%s.zca",tmp->value,md5str);
85#ifdef DEBUG
86    fprintf(stderr,"Cache list : %s\n",fname);
87    fflush(stderr);
88#endif
89    FILE* fo=fopen(fname,"w+");
90    if(fo==NULL){
91#ifdef DEBUG
92      fprintf (stderr, "Failed to open %s for writing: %s\n",fname, strerror(errno));
93#endif
94      filepath = NULL; 
95      return;
96    }
97    fwrite(content,sizeof(char),length,fo);
98    fclose(fo);
99       
100        if (filepath != NULL) {
101                strncpy(filepath, fname, max_path);
102        }       
103
104    sprintf(fname,"%s/%s.zcm",tmp->value,md5str);
105    fo=fopen(fname,"w+");
106#ifdef DEBUG
107    fprintf(stderr,"MIMETYPE: %s\n",mimeType);
108#endif
109    fwrite(mimeType,sizeof(char),strlen(mimeType),fo);
110    fclose(fo);
111
112    free(md5str);
113    free(fname);
114  }
115  else {
116          filepath = NULL;
117  }       
118}
119
120/**
121 * Verify if a url is available in the cache
122 *
123 * @param conf the maps containing the settings of the main.cfg file
124 * @param request the url
125 * @return the full name of the cached file if any, NULL in other case
126 * @warning make sure to free resources returned by this function (if not NULL)
127 */
128char* isInCache(maps* conf,char* request){
129  map* tmpM=getMapFromMaps(conf,"main","cacheDir");
130  if(tmpM!=NULL){
131    char* md5str=getMd5(request);
132#ifdef DEBUG
133    fprintf(stderr,"MD5STR : (%s)\n\n",md5str);
134#endif
135    char* fname=(char*)malloc(sizeof(char)*(strlen(tmpM->value)+strlen(md5str)+6));
136    sprintf(fname,"%s/%s.zca",tmpM->value,md5str);
137    struct stat f_status;
138    int s=stat(fname, &f_status);
139    if(s==0 && f_status.st_size>0){
140      free(md5str);
141      return fname;
142    }
143    free(md5str);
144    free(fname);
145  }
146  return NULL;
147}
148
149/**
150 * Read the downloaded file for a specific input
151 *
152 * @param m the maps containing the settings of the main.cfg file
153 * @param in the input
154 * @param index the input index
155 * @param hInternet the internet connection
156 * @param error the error map pointer
157 * @return 0 in case of success, -1 in case of failure
158 */
159int readCurrentInput(maps** m,maps** in,int* index,HINTERNET* hInternet,map** error){
160  map* tmp1;
161  char sindex[5];
162  maps* content=*in;
163  map* length=getMap(content->content,"length");
164  int shouldClean=-1;
165  if(length==NULL){
166    length=createMap("length","1");
167    shouldClean=1;
168  }
169  for(int i=0;i<atoi(length->value);i++){
170    char* fcontent;
171    char *mimeType=NULL;
172    int fsize=0;
173    char cname[15];
174    char vname[11];
175    char vname1[11];
176    char sname[9];
177    char mname[15];
178    char icname[14];
179    char xname[16];
180    char oname[12];
181    if(*index>0)
182      sprintf(vname1,"value_%d",*index);
183    else
184      sprintf(vname1,"value");
185   
186    if(i>0){
187      tmp1=getMap(content->content,cname);
188      sprintf(cname,"cache_file_%d",i);
189      sprintf(vname,"value_%d",i);
190      sprintf(sname,"size_%d",i);
191      sprintf(mname,"mimeType_%d",i);
192      sprintf(icname,"isCached_%d",i);
193      sprintf(xname,"Reference_%d",i);
194      sprintf(oname,"Order_%d",i);
195    }else{
196      sprintf(cname,"cache_file");
197      sprintf(vname,"value");
198      sprintf(sname,"size");
199      sprintf(mname,"mimeType");
200      sprintf(icname,"isCached");
201      sprintf(xname,"Reference");
202      sprintf(oname,"Order");
203    }
204   
205    map* tmap=getMap(content->content,oname);
206    sprintf(sindex,"%d",*index+1);
207    if((tmp1=getMap(content->content,xname))!=NULL && tmap!=NULL && strcasecmp(tmap->value,sindex)==0){
208     
209      if(getMap(content->content,icname)==NULL){
210        fcontent=(char*)malloc((hInternet->ihandle[*index].nDataLen+1)*sizeof(char));
211        if(fcontent == NULL){
212          errorException(*m, _("Unable to allocate memory"), "InternalError",NULL);
213          return -1;
214        }
215        size_t dwRead;
216        InternetReadFile(hInternet->ihandle[*index], 
217                         (LPVOID)fcontent, 
218                         hInternet->ihandle[*index].nDataLen, 
219                         &dwRead);
220        fcontent[hInternet->ihandle[*index].nDataLen]=0;
221        fsize=hInternet->ihandle[*index].nDataLen;
222        if(hInternet->ihandle[*index].mimeType==NULL)
223          mimeType=zStrdup("none");
224        else
225          mimeType=zStrdup(hInternet->ihandle[*index].mimeType);             
226       
227        map* tmpMap=getMapOrFill(&(*in)->content,vname,"");
228        free(tmpMap->value);
229        tmpMap->value=(char*)malloc((fsize+1)*sizeof(char));
230        if(tmpMap->value==NULL){
231          return errorException(*m, _("Unable to allocate memory"), "InternalError",NULL);
232        }
233        memcpy(tmpMap->value,fcontent,(fsize+1)*sizeof(char));
234        if(hInternet->ihandle[*index].code!=200){
235          const char *error_rep_str=_("Unable to download the file for the input <%s>, response code was : %d.");
236          char *error_msg=(char*)malloc((strlen(error_rep_str)+strlen(content->name)+4)*sizeof(char));
237          sprintf(error_msg,error_rep_str,content->name,hInternet->ihandle[*index].code);
238          if(*error==NULL){
239            *error=createMap("text",error_msg);
240            addToMap(*error,"locator",content->name);
241            addToMap(*error,"code","InvalidParameterValue");
242          }else{
243            int nb=1;
244            map* tmpMap=getMap(*error,"length");
245            if(tmpMap!=NULL)
246              nb=atoi(tmpMap->value);
247            setMapArray(*error,"text",nb,error_msg);
248            setMapArray(*error,"locator",nb,content->name);
249            setMapArray(*error,"code",nb,"InvalidParameterValue");
250          }
251          return -1;
252        }
253       
254        char ltmp1[256];
255        sprintf(ltmp1,"%d",fsize);
256        map* tmp=getMapFromMaps(*m,"main","cacheDir");
257        if(tmp!=NULL){
258          char* md5str=getMd5(tmp1->value);
259          char* fname=(char*)malloc(sizeof(char)*(strlen(tmp->value)+strlen(md5str)+6));
260          sprintf(fname,"%s/%s.zca",tmp->value,md5str);
261          addToMap((*in)->content,cname,fname);
262          free(fname);
263        }
264        addToMap((*in)->content,sname,ltmp1);
265        addToMap((*in)->content,mname,mimeType);
266        addToCache(*m,tmp1->value,fcontent,mimeType,fsize, NULL, 0);
267        free(fcontent);
268        free(mimeType);
269        *index++;
270       
271      }
272    }
273  }
274  if(shouldClean>0){
275    freeMap(&length);
276    free(length);
277  }
278  return 0;
279}
280
281/**
282 * Effectively run all the HTTP requests in the queue
283 *
284 * @param m the maps containing the settings of the main.cfg file
285 * @param inputs the maps containing the inputs (defined in the requests+added
286 *  per default based on the zcfg file)
287 * @param hInternet the HINTERNET pointer
288 * @param error the error map pointer
289 * @return 0 on success, -1 on failure
290 */
291int runHttpRequests(maps** m,maps** inputs,HINTERNET* hInternet,map** error){
292  int hasAFailure=0;
293  if(hInternet!=NULL && hInternet->nb>0){
294    AddHeaderEntries(hInternet,*m);
295    processDownloads(hInternet);
296    maps* content=*inputs;
297    int index=0;
298    while(content!=NULL){
299      if(content->child!=NULL){
300        maps* cursor=content->child;
301        while(cursor!=NULL){
302          int red=readCurrentInput(m,&cursor,&index,hInternet,error);
303          if(red<0)
304            hasAFailure=red;
305          cursor=cursor->next;
306        }
307      }
308      else{
309        int red=readCurrentInput(m,&content,&index,hInternet,error);
310        if(red<0)
311          hasAFailure=red;
312      }
313      content=content->next;
314    }
315  }
316  return hasAFailure;
317}
318
319/**
320 * Add a request in the download queue
321 *
322 * @param m the maps containing the settings of the main.cfg file
323 * @param url the url to add to the queue
324 */
325void addRequestToQueue(maps** m,HINTERNET* hInternet,const char* url,bool req){
326  hInternet->waitingRequests[hInternet->nb]=strdup(url);
327  if(req)
328    InternetOpenUrl(hInternet,hInternet->waitingRequests[hInternet->nb],NULL,0,INTERNET_FLAG_NO_CACHE_WRITE,0);
329  maps *oreq=getMaps(*m,"orequests");
330  if(oreq==NULL){
331    oreq=createMaps("orequests");
332    oreq->content=createMap("value",url);
333    addMapsToMaps(m,oreq);
334    freeMaps(&oreq);
335    free(oreq);
336  }else{
337    setMapArray(oreq->content,"value",hInternet->nb-1,url);
338  }
339}
340
341/**
342 * Try to load file from cache or download a remote file if not in cache
343 *
344 * @param m the maps containing the settings of the main.cfg file
345 * @param content the map to update
346 * @param hInternet the HINTERNET pointer
347 * @param url the url to fetch
348 * @return 0
349 */
350int loadRemoteFile(maps** m,map** content,HINTERNET* hInternet,char *url){
351  char* fcontent = NULL;
352  char* cached=isInCache(*m,url);
353  char *mimeType=NULL;
354  int fsize=0;
355
356  map* t=getMap(*content,"xlink:href");
357  if(t==NULL){
358    t=getMap((*content),"href");
359    addToMap(*content,"xlink:href",url);
360  }
361
362  if(cached!=NULL){
363
364    struct stat f_status;
365    int s=stat(cached, &f_status);
366    if(s==0){
367      fcontent=(char*)malloc(sizeof(char)*(f_status.st_size+1));
368      FILE* f=fopen(cached,"rb");
369      fread(fcontent,f_status.st_size,1,f);
370      fsize=f_status.st_size;
371      fcontent[fsize]=0;
372      fclose(f);
373      addToMap(*content,"cache_file",cached);
374    }
375    cached[strlen(cached)-1]='m';
376    s=stat(cached, &f_status);
377    if(s==0){
378      mimeType=(char*)malloc(sizeof(char)*(f_status.st_size+1));
379      FILE* f=fopen(cached,"rb");
380      fread(mimeType,f_status.st_size,1,f);
381      mimeType[f_status.st_size]=0;
382      fclose(f);
383    }
384
385  }else{   
386    addRequestToQueue(m,hInternet,url,true);
387    return 0;
388  }
389  if(fsize==0){
390    return errorException(*m, _("Unable to download the file."), "InternalError",NULL);
391  }
392  if(mimeType!=NULL){
393    addToMap(*content,"fmimeType",mimeType);
394  }
395
396  map* tmpMap=getMapOrFill(content,"value","");
397   
398  free(tmpMap->value);
399  tmpMap->value=(char*)malloc((fsize+1)*sizeof(char));
400  if(tmpMap->value==NULL || fcontent == NULL)
401    return errorException(*m, _("Unable to allocate memory"), "InternalError",NULL);
402  memcpy(tmpMap->value,fcontent,(fsize+1)*sizeof(char));
403
404  char ltmp1[256];
405  sprintf(ltmp1,"%d",fsize);
406  addToMap(*content,"size",ltmp1);
407  if(cached==NULL){
408    addToCache(*m,url,fcontent,mimeType,fsize, NULL, 0);
409  }
410  else{
411    addToMap(*content,"isCached","true");
412    map* tmp=getMapFromMaps(*m,"main","cacheDir");
413    if(tmp!=NULL){
414      map *c=getMap((*content),"xlink:href");
415      char* md5str=getMd5(c->value);
416      char* fname=(char*)malloc(sizeof(char)*(strlen(tmp->value)+strlen(md5str)+6));
417      sprintf(fname,"%s/%s.zca",tmp->value,md5str);
418      addToMap(*content,"cache_file",fname);
419      free(fname);
420    }
421  }
422  free(fcontent);
423  free(mimeType);
424  free(cached);
425  return 0;
426}
Note: See TracBrowser for help on using the repository browser.

Search

ZOO Sponsors

http://www.zoo-project.org/trac/chrome/site/img/geolabs-logo.pnghttp://www.zoo-project.org/trac/chrome/site/img/neogeo-logo.png http://www.zoo-project.org/trac/chrome/site/img/apptech-logo.png http://www.zoo-project.org/trac/chrome/site/img/3liz-logo.png http://www.zoo-project.org/trac/chrome/site/img/gateway-logo.png

Become a sponsor !

Knowledge partners

http://www.zoo-project.org/trac/chrome/site/img/ocu-logo.png http://www.zoo-project.org/trac/chrome/site/img/gucas-logo.png http://www.zoo-project.org/trac/chrome/site/img/polimi-logo.png http://www.zoo-project.org/trac/chrome/site/img/fem-logo.png http://www.zoo-project.org/trac/chrome/site/img/supsi-logo.png http://www.zoo-project.org/trac/chrome/site/img/cumtb-logo.png

Become a knowledge partner

Related links

http://zoo-project.org/img/ogclogo.png http://zoo-project.org/img/osgeologo.png