source: trunk/zoo-project/zoo-kernel/caching.c @ 695

Last change on this file since 695 was 695, checked in by djay, 4 years ago

Do not set header to NULL in addRequestToQueue, it should already contain a curl_slist. This fix issue for both Body and BodyReference?.

  • Property svn:keywords set to Id
File size: 11.0 KB
Line 
1/*
2 * Author : Gérald Fenoy
3 *
4 *  Copyright 2008-2015 GeoLabs SARL. All rights reserved.
5 *
6 * Permission is hereby granted, free of charge, to any person obtaining a copy
7 * of this software and associated documentation files (the "Software"), to deal
8 * in the Software without restriction, including without limitation the rights
9 * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
10 * copies of the Software, and to permit persons to whom the Software is
11 * furnished to do so, subject to the following conditions:
12 *
13 * The above copyright notice and this permission notice shall be included in
14 * all copies or substantial portions of the Software.
15 *
16 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
17 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
18 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
19 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
20 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
21 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
22 * THE SOFTWARE.
23 */
24
25#include "caching.h"
26#include "service.h"
27#include "service_internal.h"
28#include "response_print.h"
29#include <openssl/md5.h>
30#include <openssl/hmac.h>
31#include <openssl/evp.h>
32#include <openssl/bio.h>
33#include <openssl/buffer.h>
34
35/**
36 * Compute md5
37 *
38 * @param url the char*
39 * @return a char* representing the md5 of the url
40 * @warning make sure to free ressources returned by this function
41 */
42char* getMd5(char* url){
43  EVP_MD_CTX md5ctx;
44  char* fresult=(char*)malloc((EVP_MAX_MD_SIZE+1)*sizeof(char));
45  unsigned char result[EVP_MAX_MD_SIZE];
46  unsigned int len;
47  EVP_DigestInit(&md5ctx, EVP_md5());
48  EVP_DigestUpdate(&md5ctx, url, strlen(url));
49  EVP_DigestFinal_ex(&md5ctx,result,&len);
50  EVP_MD_CTX_cleanup(&md5ctx);
51  int i;
52  for(i = 0; i < len; i++){
53    if(i>0){
54      char *tmp=strdup(fresult);
55      sprintf(fresult,"%s%02x", tmp,result[i]);
56      free(tmp);
57    }
58    else
59      sprintf(fresult,"%02x",result[i]);
60  }
61  return fresult;
62}
63
64
65/**
66 * Cache a file for a given request.
67 * For each cached file, the are two files stored, a .zca and a .zcm containing
68 * the downloaded content and the mimeType respectively.
69 *
70 * @param conf the maps containing the settings of the main.cfg file
71 * @param request the url used too fetch the content
72 * @param content the downloaded content
73 * @param mimeType the content mimeType
74 * @param length the content size
75 * @param filepath a buffer for storing the path of the cached file; may be NULL
76 * @param max_path the size of the allocated filepath buffer 
77 */
78void addToCache(maps* conf,char* request,char* content,char* mimeType,int length, 
79                char* filepath, size_t max_path){
80  map* tmp=getMapFromMaps(conf,"main","cacheDir");
81  if(tmp!=NULL){
82    char* md5str=getMd5(request);
83    char* fname=(char*)malloc(sizeof(char)*(strlen(tmp->value)+strlen(md5str)+6));
84    sprintf(fname,"%s/%s.zca",tmp->value,md5str);
85#ifdef DEBUG
86    fprintf(stderr,"Cache list : %s\n",fname);
87    fflush(stderr);
88#endif
89    FILE* fo=fopen(fname,"w+");
90    if(fo==NULL){
91#ifdef DEBUG
92      fprintf (stderr, "Failed to open %s for writing: %s\n",fname, strerror(errno));
93#endif
94      filepath = NULL; 
95      return;
96    }
97    fwrite(content,sizeof(char),length,fo);
98    fclose(fo);
99       
100        if (filepath != NULL) {
101                strncpy(filepath, fname, max_path);
102        }       
103
104    sprintf(fname,"%s/%s.zcm",tmp->value,md5str);
105    fo=fopen(fname,"w+");
106#ifdef DEBUG
107    fprintf(stderr,"MIMETYPE: %s\n",mimeType);
108#endif
109    fwrite(mimeType,sizeof(char),strlen(mimeType),fo);
110    fclose(fo);
111
112    free(md5str);
113    free(fname);
114  }
115  else {
116          filepath = NULL;
117  }       
118}
119
120/**
121 * Verify if a url is available in the cache
122 *
123 * @param conf the maps containing the settings of the main.cfg file
124 * @param request the url
125 * @return the full name of the cached file if any, NULL in other case
126 * @warning make sure to free ressources returned by this function (if not NULL)
127 */
128char* isInCache(maps* conf,char* request){
129  map* tmpM=getMapFromMaps(conf,"main","cacheDir");
130  if(tmpM!=NULL){
131    char* md5str=getMd5(request);
132#ifdef DEBUG
133    fprintf(stderr,"MD5STR : (%s)\n\n",md5str);
134#endif
135    char* fname=(char*)malloc(sizeof(char)*(strlen(tmpM->value)+strlen(md5str)+6));
136    sprintf(fname,"%s/%s.zca",tmpM->value,md5str);
137    struct stat f_status;
138    int s=stat(fname, &f_status);
139    if(s==0 && f_status.st_size>0){
140      free(md5str);
141      return fname;
142    }
143    free(md5str);
144    free(fname);
145  }
146  return NULL;
147}
148
149/**
150 * Effectively run all the HTTP requests in the queue
151 *
152 * @param m the maps containing the settings of the main.cfg file
153 * @param inputs the maps containing the inputs (defined in the requests+added
154 *  per default based on the zcfg file)
155 * @param hInternet the HINTERNET pointer
156 * @return 0 on success
157 */
158int runHttpRequests(maps** m,maps** inputs,HINTERNET* hInternet){
159  if(hInternet->nb>0){
160    processDownloads(hInternet);
161    maps* content=*inputs;
162    map* tmp1;
163    int index=0;
164    char sindex[5];
165    while(content!=NULL){
166     
167      map* length=getMap(content->content,"length");
168      int shouldClean=-1;
169      if(length==NULL){
170        length=createMap("length","1");
171        shouldClean=1;
172      }
173      for(int i=0;i<atoi(length->value);i++){
174        char* fcontent;
175        char *mimeType=NULL;
176        int fsize=0;
177        char cname[15];
178        char vname[11];
179        char vname1[11];
180        char sname[9];
181        char mname[15];
182        char icname[14];
183        char xname[16];
184        char oname[12];
185        if(index>0)
186          sprintf(vname1,"value_%d",index);
187        else
188          sprintf(vname1,"value");
189
190        if(i>0){
191          tmp1=getMap(content->content,cname);
192          sprintf(cname,"cache_file_%d",i);
193          sprintf(vname,"value_%d",i);
194          sprintf(sname,"size_%d",i);
195          sprintf(mname,"mimeType_%d",i);
196          sprintf(icname,"isCached_%d",i);
197          sprintf(xname,"Reference_%d",i);
198          sprintf(oname,"Order_%d",i);
199        }else{
200          sprintf(cname,"cache_file");
201          sprintf(vname,"value");
202          sprintf(sname,"size");
203          sprintf(mname,"mimeType");
204          sprintf(icname,"isCached");
205          sprintf(xname,"Reference");
206          sprintf(oname,"Order");
207        }
208
209        map* tmap=getMap(content->content,oname);
210        sprintf(sindex,"%d",index+1);
211        if((tmp1=getMap(content->content,xname))!=NULL && tmap!=NULL && strcasecmp(tmap->value,sindex)==0){
212
213          if(getMap(content->content,icname)==NULL){
214           
215            fcontent=(char*)malloc((hInternet->ihandle[index].nDataLen+1)*sizeof(char));
216            if(fcontent == NULL){
217              return errorException(*m, _("Unable to allocate memory."), "InternalError",NULL);
218            }
219            size_t dwRead;
220            InternetReadFile(hInternet->ihandle[index], 
221                             (LPVOID)fcontent, 
222                             hInternet->ihandle[index].nDataLen, 
223                             &dwRead);
224            fcontent[hInternet->ihandle[index].nDataLen]=0;
225            fsize=hInternet->ihandle[index].nDataLen;
226            if(hInternet->ihandle[index].mimeType==NULL)
227              mimeType=strdup("none");
228            else
229              mimeType=strdup(hInternet->ihandle[index].mimeType);           
230           
231            map* tmpMap=getMapOrFill(&content->content,vname,"");
232            free(tmpMap->value);
233            tmpMap->value=(char*)malloc((fsize+1)*sizeof(char));
234            if(tmpMap->value==NULL){
235              return errorException(*m, _("Unable to allocate memory."), "InternalError",NULL);
236            }
237            memcpy(tmpMap->value,fcontent,(fsize+1)*sizeof(char));
238           
239            char ltmp1[256];
240            sprintf(ltmp1,"%d",fsize);
241            map* tmp=getMapFromMaps(*m,"main","cacheDir");
242            if(tmp!=NULL){
243              char* md5str=getMd5(tmp1->value);
244              char* fname=(char*)malloc(sizeof(char)*(strlen(tmp->value)+strlen(md5str)+6));
245              sprintf(fname,"%s/%s.zca",tmp->value,md5str);
246              addToMap(content->content,cname,fname);
247              free(fname);
248            }
249            addToMap(content->content,sname,ltmp1);
250            addToMap(content->content,mname,mimeType);
251            addToCache(*m,tmp1->value,fcontent,mimeType,fsize, NULL, 0);
252            free(fcontent);
253            free(mimeType);
254            index++;
255
256          }
257        }
258      }
259      if(shouldClean>0){
260        freeMap(&length);
261        free(length);
262      }
263     
264      content=content->next;
265    }
266   
267  }
268  return 0;
269}
270
271/**
272 * Add a request in the download queue
273 *
274 * @param m the maps containing the settings of the main.cfg file
275 * @param url the url to add to the queue
276 */
277void addRequestToQueue(maps** m,HINTERNET* hInternet,const char* url,bool req){
278  hInternet->waitingRequests[hInternet->nb]=strdup(url);
279  if(req)
280    InternetOpenUrl(hInternet,hInternet->waitingRequests[hInternet->nb],NULL,0,INTERNET_FLAG_NO_CACHE_WRITE,0);
281  maps *oreq=getMaps(*m,"orequests");
282  if(oreq==NULL){
283    oreq=(maps*)malloc(MAPS_SIZE);
284    oreq->name=zStrdup("orequests");
285    oreq->content=createMap("value",url);
286    oreq->next=NULL;
287    addMapsToMaps(m,oreq);
288    freeMaps(&oreq);
289    free(oreq);
290  }else{
291    setMapArray(oreq->content,"value",hInternet->nb-1,url);
292  }
293}
294
295/**
296 * Try to load file from cache or download a remote file if not in cache
297 *
298 * @param m the maps containing the settings of the main.cfg file
299 * @param content the map to update
300 * @param hInternet the HINTERNET pointer
301 * @param url the url to fetch
302 * @return 0
303 */
304int loadRemoteFile(maps** m,map** content,HINTERNET* hInternet,char *url){
305  char* fcontent;
306  char* cached=isInCache(*m,url);
307  char *mimeType=NULL;
308  int fsize=0;
309
310  map* t=getMap(*content,"xlink:href");
311  if(t==NULL){
312    t=getMap((*content),"href");
313    addToMap(*content,"xlink:href",url);
314  }
315
316  if(cached!=NULL){
317
318    struct stat f_status;
319    int s=stat(cached, &f_status);
320    if(s==0){
321      fcontent=(char*)malloc(sizeof(char)*(f_status.st_size+1));
322      FILE* f=fopen(cached,"rb");
323      fread(fcontent,f_status.st_size,1,f);
324      fsize=f_status.st_size;
325      fcontent[fsize]=0;
326      fclose(f);
327      addToMap(*content,"cache_file",cached);
328    }
329    cached[strlen(cached)-1]='m';
330    s=stat(cached, &f_status);
331    if(s==0){
332      mimeType=(char*)malloc(sizeof(char)*(f_status.st_size+1));
333      FILE* f=fopen(cached,"rb");
334      fread(mimeType,f_status.st_size,1,f);
335      mimeType[f_status.st_size]=0;
336      fclose(f);
337    }
338
339  }else{   
340    addRequestToQueue(m,hInternet,url,true);
341    return 0;
342  }
343  if(fsize==0){
344    return errorException(*m, _("Unable to download the file."), "InternalError",NULL);
345  }
346  if(mimeType!=NULL){
347    addToMap(*content,"fmimeType",mimeType);
348  }
349
350  map* tmpMap=getMapOrFill(content,"value","");
351   
352  free(tmpMap->value);
353
354  tmpMap->value=(char*)malloc((fsize+1)*sizeof(char));
355  if(tmpMap->value==NULL)
356    return errorException(*m, _("Unable to allocate memory."), "InternalError",NULL);
357  memcpy(tmpMap->value,fcontent,(fsize+1)*sizeof(char));
358
359  char ltmp1[256];
360  sprintf(ltmp1,"%d",fsize);
361  addToMap(*content,"size",ltmp1);
362  if(cached==NULL){
363    addToCache(*m,url,fcontent,mimeType,fsize, NULL, 0);
364  }
365  else{
366    addToMap(*content,"isCached","true");
367    map* tmp=getMapFromMaps(*m,"main","cacheDir");
368    if(tmp!=NULL){
369      map *c=getMap((*content),"xlink:href");
370      char* md5str=getMd5(c->value);
371      char* fname=(char*)malloc(sizeof(char)*(strlen(tmp->value)+strlen(md5str)+6));
372      sprintf(fname,"%s/%s.zca",tmp->value,md5str);
373      addToMap(*content,"cache_file",fname);
374      free(fname);
375    }
376  }
377  free(fcontent);
378  free(mimeType);
379  free(cached);
380  return 0;
381}
Note: See TracBrowser for help on using the repository browser.

Search

ZOO Sponsors

http://www.zoo-project.org/trac/chrome/site/img/geolabs-logo.pnghttp://www.zoo-project.org/trac/chrome/site/img/neogeo-logo.png http://www.zoo-project.org/trac/chrome/site/img/apptech-logo.png http://www.zoo-project.org/trac/chrome/site/img/3liz-logo.png http://www.zoo-project.org/trac/chrome/site/img/gateway-logo.png

Become a sponsor !

Knowledge partners

http://www.zoo-project.org/trac/chrome/site/img/ocu-logo.png http://www.zoo-project.org/trac/chrome/site/img/gucas-logo.png http://www.zoo-project.org/trac/chrome/site/img/polimi-logo.png http://www.zoo-project.org/trac/chrome/site/img/fem-logo.png http://www.zoo-project.org/trac/chrome/site/img/supsi-logo.png http://www.zoo-project.org/trac/chrome/site/img/cumtb-logo.png

Become a knowledge partner

Related links

http://zoo-project.org/img/ogclogo.png http://zoo-project.org/img/osgeologo.png