source: trunk/zoo-project/zoo-kernel/caching.c @ 790

Last change on this file since 790 was 790, checked in by djay, 5 years ago

Add support for nested inputs and outputs.

  • Property svn:keywords set to Id
File size: 11.5 KB
Line 
1/*
2 * Author : Gérald Fenoy
3 *
4 *  Copyright 2008-2015 GeoLabs SARL. All rights reserved.
5 *
6 * Permission is hereby granted, free of charge, to any person obtaining a copy
7 * of this software and associated documentation files (the "Software"), to deal
8 * in the Software without restriction, including without limitation the rights
9 * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
10 * copies of the Software, and to permit persons to whom the Software is
11 * furnished to do so, subject to the following conditions:
12 *
13 * The above copyright notice and this permission notice shall be included in
14 * all copies or substantial portions of the Software.
15 *
16 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
17 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
18 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
19 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
20 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
21 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
22 * THE SOFTWARE.
23 */
24
25#include "caching.h"
26#include "service.h"
27#include "service_internal.h"
28#include "response_print.h"
29#include <openssl/md5.h>
30#include <openssl/hmac.h>
31#include <openssl/evp.h>
32#include <openssl/bio.h>
33#include <openssl/buffer.h>
34
35/**
36 * Compute md5
37 *
38 * @param url the char*
39 * @return a char* representing the md5 of the url
40 * @warning make sure to free resources returned by this function
41 */
42char* getMd5(char* url){
43  EVP_MD_CTX md5ctx;
44  char* fresult=(char*)malloc((EVP_MAX_MD_SIZE+1)*sizeof(char));
45  unsigned char result[EVP_MAX_MD_SIZE];
46  unsigned int len;
47  EVP_DigestInit(&md5ctx, EVP_md5());
48  EVP_DigestUpdate(&md5ctx, url, strlen(url));
49  EVP_DigestFinal_ex(&md5ctx,result,&len);
50  EVP_MD_CTX_cleanup(&md5ctx);
51  int i;
52  for(i = 0; i < len; i++){
53    if(i>0){
54      char *tmp=strdup(fresult);
55      sprintf(fresult,"%s%02x", tmp,result[i]);
56      free(tmp);
57    }
58    else
59      sprintf(fresult,"%02x",result[i]);
60  }
61  return fresult;
62}
63
64
65/**
66 * Cache a file for a given request.
67 * For each cached file, the are two files stored, a .zca and a .zcm containing
68 * the downloaded content and the mimeType respectively.
69 *
70 * @param conf the maps containing the settings of the main.cfg file
71 * @param request the url used too fetch the content
72 * @param content the downloaded content
73 * @param mimeType the content mimeType
74 * @param length the content size
75 * @param filepath a buffer for storing the path of the cached file; may be NULL
76 * @param max_path the size of the allocated filepath buffer 
77 */
78void addToCache(maps* conf,char* request,char* content,char* mimeType,int length, 
79                char* filepath, size_t max_path){
80  map* tmp=getMapFromMaps(conf,"main","cacheDir");
81  if(tmp!=NULL){
82    char* md5str=getMd5(request);
83    char* fname=(char*)malloc(sizeof(char)*(strlen(tmp->value)+strlen(md5str)+6));
84    sprintf(fname,"%s/%s.zca",tmp->value,md5str);
85#ifdef DEBUG
86    fprintf(stderr,"Cache list : %s\n",fname);
87    fflush(stderr);
88#endif
89    FILE* fo=fopen(fname,"w+");
90    if(fo==NULL){
91#ifdef DEBUG
92      fprintf (stderr, "Failed to open %s for writing: %s\n",fname, strerror(errno));
93#endif
94      filepath = NULL; 
95      return;
96    }
97    fwrite(content,sizeof(char),length,fo);
98    fclose(fo);
99       
100        if (filepath != NULL) {
101                strncpy(filepath, fname, max_path);
102        }       
103
104    sprintf(fname,"%s/%s.zcm",tmp->value,md5str);
105    fo=fopen(fname,"w+");
106#ifdef DEBUG
107    fprintf(stderr,"MIMETYPE: %s\n",mimeType);
108#endif
109    fwrite(mimeType,sizeof(char),strlen(mimeType),fo);
110    fclose(fo);
111
112    free(md5str);
113    free(fname);
114  }
115  else {
116          filepath = NULL;
117  }       
118}
119
120/**
121 * Verify if a url is available in the cache
122 *
123 * @param conf the maps containing the settings of the main.cfg file
124 * @param request the url
125 * @return the full name of the cached file if any, NULL in other case
126 * @warning make sure to free resources returned by this function (if not NULL)
127 */
128char* isInCache(maps* conf,char* request){
129  map* tmpM=getMapFromMaps(conf,"main","cacheDir");
130  if(tmpM!=NULL){
131    char* md5str=getMd5(request);
132#ifdef DEBUG
133    fprintf(stderr,"MD5STR : (%s)\n\n",md5str);
134#endif
135    char* fname=(char*)malloc(sizeof(char)*(strlen(tmpM->value)+strlen(md5str)+6));
136    sprintf(fname,"%s/%s.zca",tmpM->value,md5str);
137    struct stat f_status;
138    int s=stat(fname, &f_status);
139    if(s==0 && f_status.st_size>0){
140      free(md5str);
141      return fname;
142    }
143    free(md5str);
144    free(fname);
145  }
146  return NULL;
147}
148
149/**
150 * Read the downloaded file for a specific input
151 *
152 * @param m the maps containing the settings of the main.cfg file
153 * @param in the input
154 * @param index the input index
155 * @param hInternet the internet connection
156 * @return 0 in case of success, 4 in case of failure
157 */
158int readCurrentInput(maps** m,maps** in,int* index,HINTERNET* hInternet){
159  map* tmp1;
160  char sindex[5];
161  maps* content=*in;
162  map* length=getMap(content->content,"length");
163  int shouldClean=-1;
164  if(length==NULL){
165    length=createMap("length","1");
166    shouldClean=1;
167  }
168  for(int i=0;i<atoi(length->value);i++){
169    char* fcontent;
170    char *mimeType=NULL;
171    int fsize=0;
172    char cname[15];
173    char vname[11];
174    char vname1[11];
175    char sname[9];
176    char mname[15];
177    char icname[14];
178    char xname[16];
179    char oname[12];
180    if(*index>0)
181      sprintf(vname1,"value_%d",*index);
182    else
183      sprintf(vname1,"value");
184   
185    if(i>0){
186      tmp1=getMap(content->content,cname);
187      sprintf(cname,"cache_file_%d",i);
188      sprintf(vname,"value_%d",i);
189      sprintf(sname,"size_%d",i);
190      sprintf(mname,"mimeType_%d",i);
191      sprintf(icname,"isCached_%d",i);
192      sprintf(xname,"Reference_%d",i);
193      sprintf(oname,"Order_%d",i);
194    }else{
195      sprintf(cname,"cache_file");
196      sprintf(vname,"value");
197      sprintf(sname,"size");
198      sprintf(mname,"mimeType");
199      sprintf(icname,"isCached");
200      sprintf(xname,"Reference");
201      sprintf(oname,"Order");
202    }
203   
204    map* tmap=getMap(content->content,oname);
205    sprintf(sindex,"%d",*index+1);
206    if((tmp1=getMap(content->content,xname))!=NULL && tmap!=NULL && strcasecmp(tmap->value,sindex)==0){
207     
208      if(getMap(content->content,icname)==NULL){
209       
210        fcontent=(char*)malloc((hInternet->ihandle[*index].nDataLen+1)*sizeof(char));
211        if(fcontent == NULL){
212          return errorException(*m, _("Unable to allocate memory"), "InternalError",NULL);
213        }
214        size_t dwRead;
215        InternetReadFile(hInternet->ihandle[*index], 
216                         (LPVOID)fcontent, 
217                         hInternet->ihandle[*index].nDataLen, 
218                         &dwRead);
219        fcontent[hInternet->ihandle[*index].nDataLen]=0;
220        fsize=hInternet->ihandle[*index].nDataLen;
221        if(hInternet->ihandle[*index].mimeType==NULL)
222          mimeType=zStrdup("none");
223        else
224          mimeType=zStrdup(hInternet->ihandle[*index].mimeType);             
225       
226        map* tmpMap=getMapOrFill(&(*in)->content,vname,"");
227        free(tmpMap->value);
228        tmpMap->value=(char*)malloc((fsize+1)*sizeof(char));
229        if(tmpMap->value==NULL){
230          return errorException(*m, _("Unable to allocate memory"), "InternalError",NULL);
231        }
232        memcpy(tmpMap->value,fcontent,(fsize+1)*sizeof(char));
233       
234        char ltmp1[256];
235        sprintf(ltmp1,"%d",fsize);
236        map* tmp=getMapFromMaps(*m,"main","cacheDir");
237        if(tmp!=NULL){
238          char* md5str=getMd5(tmp1->value);
239          char* fname=(char*)malloc(sizeof(char)*(strlen(tmp->value)+strlen(md5str)+6));
240          sprintf(fname,"%s/%s.zca",tmp->value,md5str);
241          addToMap((*in)->content,cname,fname);
242          free(fname);
243        }
244        addToMap((*in)->content,sname,ltmp1);
245        addToMap((*in)->content,mname,mimeType);
246        addToCache(*m,tmp1->value,fcontent,mimeType,fsize, NULL, 0);
247        free(fcontent);
248        free(mimeType);
249        *index++;
250       
251      }
252    }
253  }
254  if(shouldClean>0){
255    freeMap(&length);
256    free(length);
257  }
258  return 0;
259}
260
261/**
262 * Effectively run all the HTTP requests in the queue
263 *
264 * @param m the maps containing the settings of the main.cfg file
265 * @param inputs the maps containing the inputs (defined in the requests+added
266 *  per default based on the zcfg file)
267 * @param hInternet the HINTERNET pointer
268 * @return 0 on success
269 */
270int runHttpRequests(maps** m,maps** inputs,HINTERNET* hInternet){
271  if(hInternet!=NULL && hInternet->nb>0){
272    processDownloads(hInternet);
273    maps* content=*inputs;
274    int index=0;
275    while(content!=NULL){
276      if(content->child!=NULL){
277        maps* cursor=content->child;
278        while(cursor!=NULL){
279          readCurrentInput(m,&cursor,&index,hInternet);
280          cursor=cursor->next;
281        }
282      }
283      else
284        readCurrentInput(m,&content,&index,hInternet);
285      content=content->next;
286    } 
287  }
288  return 0;
289}
290
291/**
292 * Add a request in the download queue
293 *
294 * @param m the maps containing the settings of the main.cfg file
295 * @param url the url to add to the queue
296 */
297void addRequestToQueue(maps** m,HINTERNET* hInternet,const char* url,bool req){
298  hInternet->waitingRequests[hInternet->nb]=strdup(url);
299  if(req)
300    InternetOpenUrl(hInternet,hInternet->waitingRequests[hInternet->nb],NULL,0,INTERNET_FLAG_NO_CACHE_WRITE,0);
301  maps *oreq=getMaps(*m,"orequests");
302  if(oreq==NULL){
303    oreq=createMaps("orequests");
304    oreq->content=createMap("value",url);
305    addMapsToMaps(m,oreq);
306    freeMaps(&oreq);
307    free(oreq);
308  }else{
309    setMapArray(oreq->content,"value",hInternet->nb-1,url);
310  }
311}
312
313/**
314 * Try to load file from cache or download a remote file if not in cache
315 *
316 * @param m the maps containing the settings of the main.cfg file
317 * @param content the map to update
318 * @param hInternet the HINTERNET pointer
319 * @param url the url to fetch
320 * @return 0
321 */
322int loadRemoteFile(maps** m,map** content,HINTERNET* hInternet,char *url){
323  char* fcontent = NULL;
324  char* cached=isInCache(*m,url);
325  char *mimeType=NULL;
326  int fsize=0;
327
328  map* t=getMap(*content,"xlink:href");
329  if(t==NULL){
330    t=getMap((*content),"href");
331    addToMap(*content,"xlink:href",url);
332  }
333
334  if(cached!=NULL){
335
336    struct stat f_status;
337    int s=stat(cached, &f_status);
338    if(s==0){
339      fcontent=(char*)malloc(sizeof(char)*(f_status.st_size+1));
340      FILE* f=fopen(cached,"rb");
341      fread(fcontent,f_status.st_size,1,f);
342      fsize=f_status.st_size;
343      fcontent[fsize]=0;
344      fclose(f);
345      addToMap(*content,"cache_file",cached);
346    }
347    cached[strlen(cached)-1]='m';
348    s=stat(cached, &f_status);
349    if(s==0){
350      mimeType=(char*)malloc(sizeof(char)*(f_status.st_size+1));
351      FILE* f=fopen(cached,"rb");
352      fread(mimeType,f_status.st_size,1,f);
353      mimeType[f_status.st_size]=0;
354      fclose(f);
355    }
356
357  }else{   
358    addRequestToQueue(m,hInternet,url,true);
359    return 0;
360  }
361  if(fsize==0){
362    return errorException(*m, _("Unable to download the file."), "InternalError",NULL);
363  }
364  if(mimeType!=NULL){
365    addToMap(*content,"fmimeType",mimeType);
366  }
367
368  map* tmpMap=getMapOrFill(content,"value","");
369   
370  free(tmpMap->value);
371  tmpMap->value=(char*)malloc((fsize+1)*sizeof(char));
372  if(tmpMap->value==NULL || fcontent == NULL)
373    return errorException(*m, _("Unable to allocate memory"), "InternalError",NULL);
374  memcpy(tmpMap->value,fcontent,(fsize+1)*sizeof(char));
375
376  char ltmp1[256];
377  sprintf(ltmp1,"%d",fsize);
378  addToMap(*content,"size",ltmp1);
379  if(cached==NULL){
380    addToCache(*m,url,fcontent,mimeType,fsize, NULL, 0);
381  }
382  else{
383    addToMap(*content,"isCached","true");
384    map* tmp=getMapFromMaps(*m,"main","cacheDir");
385    if(tmp!=NULL){
386      map *c=getMap((*content),"xlink:href");
387      char* md5str=getMd5(c->value);
388      char* fname=(char*)malloc(sizeof(char)*(strlen(tmp->value)+strlen(md5str)+6));
389      sprintf(fname,"%s/%s.zca",tmp->value,md5str);
390      addToMap(*content,"cache_file",fname);
391      free(fname);
392    }
393  }
394  free(fcontent);
395  free(mimeType);
396  free(cached);
397  return 0;
398}
Note: See TracBrowser for help on using the repository browser.

Search

ZOO Sponsors

http://www.zoo-project.org/trac/chrome/site/img/geolabs-logo.pnghttp://www.zoo-project.org/trac/chrome/site/img/neogeo-logo.png http://www.zoo-project.org/trac/chrome/site/img/apptech-logo.png http://www.zoo-project.org/trac/chrome/site/img/3liz-logo.png http://www.zoo-project.org/trac/chrome/site/img/gateway-logo.png

Become a sponsor !

Knowledge partners

http://www.zoo-project.org/trac/chrome/site/img/ocu-logo.png http://www.zoo-project.org/trac/chrome/site/img/gucas-logo.png http://www.zoo-project.org/trac/chrome/site/img/polimi-logo.png http://www.zoo-project.org/trac/chrome/site/img/fem-logo.png http://www.zoo-project.org/trac/chrome/site/img/supsi-logo.png http://www.zoo-project.org/trac/chrome/site/img/cumtb-logo.png

Become a knowledge partner

Related links

http://zoo-project.org/img/ogclogo.png http://zoo-project.org/img/osgeologo.png