source: trunk/zoo-project/zoo-kernel/ulinet.c @ 621

Last change on this file since 621 was 621, checked in by djay, 9 years ago

Major update. Creation of a basic parsing api. Call validateRequest after fork if any.

  • Property svn:eol-style set to native
  • Property svn:mime-type set to text/x-csrc
File size: 14.8 KB
RevLine 
[579]1/*
[1]2 *  ulinet.c
3 *
4 * Author : Gérald FENOY
5 *
[579]6 * Copyright (c) 2008-2015 GeoLabs SARL
[1]7 *
8 * Permission is hereby granted, free of charge, to any person obtaining a copy
9 * of this software and associated documentation files (the "Software"), to deal
10 * in the Software without restriction, including without limitation the rights
11 * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
12 * copies of the Software, and to permit persons to whom the Software is
13 * furnished to do so, subject to the following conditions:
14 *
15 * The above copyright notice and this permission notice shall be included in
16 * all copies or substantial portions of the Software.
17 *
18 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
19 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
20 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
21 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
22 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
23 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
24 * THE SOFTWARE.
25 *
26 */
27
28#define _ULINET
[490]29#define MAX_WAIT_MSECS 180*1000 /* Wait max. 180 seconds */
[1]30#include "ulinet.h"
31#include <assert.h>
[579]32
33/**
34 * Write the downloaded content to a _HINTERNET structure
35 *
36 * @param buffer the buffer to read
37 * @param size size of each member
38 * @param nmemb number of element to read
39 * @param data the _HINTERNET structure to write in
40 * @return the size red, -1 if buffer is NULL
41 */
[1]42size_t write_data_into(void *buffer, size_t size, size_t nmemb, void *data){
43  size_t realsize = size * nmemb;
[492]44  _HINTERNET *psInternet;
[93]45  if(buffer==NULL){
[1]46    buffer=NULL;
47    return -1;
48  }
[492]49  psInternet=(_HINTERNET *)data;
[1]50  if(psInternet->pabyData){
[446]51    psInternet->pabyData=(unsigned char*)realloc(psInternet->pabyData,psInternet->nDataLen+realsize+1);
[1]52    psInternet->nDataAlloc+=psInternet->nDataLen+realsize+1;
53  }
54  else{
[446]55    psInternet->pabyData=(unsigned char*)malloc(psInternet->nDataLen+realsize+1);
[1]56    psInternet->nDataAlloc=realsize+1;
57  }
58
59  if (psInternet->pabyData) {
60    memcpy( psInternet->pabyData + psInternet->nDataLen, buffer, realsize);
61    psInternet->nDataLen += realsize;
62    psInternet->pabyData[psInternet->nDataLen] = 0;
63  }
64
65  buffer=NULL;
66  return realsize;
67}
68
[579]69/**
70 * In case of presence of "Set-Cookie" in the headers red, store the cookie
71 * identifier in CCookie
72 *
73 * @param buffer the buffer to read
74 * @param size size of each member
75 * @param nmemb number of element to read
76 * @param data the _HINTERNET structure to write in
77 * @return the size red, -1 if buffer is NULL
78 * @see CCookie
79 */
[1]80size_t header_write_data(void *buffer, size_t size, size_t nmemb, void *data){
[509]81  if(strncmp("Set-Cookie: ",(char*)buffer,12)==0){
[1]82    int i;
[508]83    char env[256];
84    char path[256];
85    char domain[256];
[492]86    char* tmp;
[1]87    for(i=0;i<12;i++)
88#ifndef WIN32
89      buffer++;
90#else
91        ;
92#endif
[509]93    sscanf((char*)buffer,"%s; path=%s; domain=%s",env,path,domain);
94    tmp=strcat(env,CCookie[0]);
[1]95#ifdef MSG_LAF_OUT
96    printf("\n**Cookie env : [%s] , path : [%s], domain : [%s]**\n",env,path,domain);
97    printf("buffer : %d (%s) (%s) (%s)\n",(buffer==NULL),buffer,tmp,CCookie);
98#endif
[509]99    strcpy(CCookie[0],tmp);
[1]100  }
101  return size * nmemb;//write_data_into(buffer,size,nmemb,data,HEADER);
102};
103
[579]104/**
105 * Define the proxy to use for a CURL handler
106 *
107 * @param handle the CURL handler
108 * @param host the proxy host (including http://)
109 * @param port the proxy port
110 */
[1]111void setProxy(CURL* handle,char* host,long port){
[579]112  char* proxyDef=(char*)malloc((strlen(host)+10+2)*sizeof(char));
113  sprintf(proxyDef,"%s:%ld",host,port);
114  curl_easy_setopt(handle,CURLOPT_PROXY,proxyDef);
115  free(proxyDef);
[1]116}
117
118/**
119 * MACOSX
120 */
121#if defined(macintosh) || (defined(__MACH__) && defined(__APPLE__))
122
123
124char* CFStringToCString(CFStringRef dest,char *buffer){
125  CFStringEncoding encoding = kCFStringEncodingUTF8;
126  Boolean bool2 = CFStringGetCString(dest,buffer,1024,encoding);
127  if(bool2){
128    printf("Loaded into local_buffer");
129    return buffer;
130  }
131  return NULL;
132}
133
134OSStatus setProxiesForProtcol(CURL* handle,const char *proto){
135  OSStatus              err;
136  CFDictionaryRef proxyDict;
137  CFArrayRef            proxies;
138 
[446]139  CFStringRef key_enabled = NULL;
140  CFStringRef key_host = NULL;
141  CFStringRef key_port = NULL;
[1]142 
143  bool proxy_enabled;
144  char *proxy_host;
145  long proxy_port;
146 
147  proxyDict = NULL;
148  proxies = NULL;
149
150  err = noErr;
151  proxyDict = SCDynamicStoreCopyProxies(NULL);
152
[446]153  if(strncmp(proto,"http",4)==0){
[1]154      key_enabled=kSCPropNetProxiesHTTPEnable;
155      key_host=kSCPropNetProxiesHTTPProxy;
156      key_port=kSCPropNetProxiesHTTPPort;
157  }
158  else
[446]159    if(strncmp(proto,"https",5)==0){
[1]160      key_enabled=kSCPropNetProxiesHTTPSEnable;
161      key_host=kSCPropNetProxiesHTTPSProxy;
162      key_port=kSCPropNetProxiesHTTPSPort;
163    }
164
165  CFNumberGetValue(CFDictionaryGetValue(proxyDict,key_enabled),kCFNumberIntType,&proxy_enabled);
166  if(proxy_enabled){
167    CFNumberGetValue(CFDictionaryGetValue(proxyDict,key_port),CFNumberGetType(CFDictionaryGetValue(proxyDict,key_port)),&proxy_port);
168    char buffer[1024];
169    CFStringToCString(CFDictionaryGetValue(proxyDict,key_host),buffer);
170    proxy_host=buffer;
171
172#ifdef MSG_LAF_VERBOSE
173    printf("\n**[PROXY SETTINGS DETECTION %s (%d) %s:%li (%s)]**\n",proto,proxy_enabled,(char*)proxy_host,proxy_port,buffer);
174#endif
175
176    if (proxyDict == NULL) {
177      err = coreFoundationUnknownErr;
178    }
179
180    setProxy(handle,proxy_host,proxy_port);
181  }
182  return err;
183}
184#else
185/**
[579]186 * Should autodetect the proxy configuration (do nothing on linux)
187 *
188 * @param handle a CURL handle
189 * @param proto the protocol requiring the use of a proxy
[1]190 */
191bool setProxiesForProtcol(CURL* handle,const char *proto){
192#ifdef MSG_LAF_VERBOSE
193  fprintf( stderr, "setProxiesForProtocol (do nothing) ...\n" );
194#endif
[509]195  return true;
[1]196}
197#endif
198
[579]199/**
200 * Create a HINTERNET
201 *
202 * @param lpszAgent the HTPP User-Agent to use to send requests
203 * @param  dwAccessType type of access required
204 * @param  lpszProxyName the name of the proxy server(s) to use
205 * @param  lpszProxyBypass ip address or host names which should not be routed
206 *  through the proxy
207 * @param  dwFlags Options (INTERNET_FLAG_ASYNC,INTERNET_FLAG_FROM_CACHE,INTERNET_FLAG_OFFLINE)
208 * @return the created HINTERNET
209 */
[492]210HINTERNET InternetOpen(char* lpszAgent,int dwAccessType,char* lpszProxyName,char* lpszProxyBypass,int dwFlags){
[1]211  HINTERNET ret;
[490]212  ret.handle=curl_multi_init();
213  ret.agent=strdup(lpszAgent);
[492]214  ret.nb=0;
215  ret.ihandle[ret.nb].header=NULL;
[1]216  return ret;
217}
218
[579]219/**
220 * Close a HINTERNET connection and free allocated ressources
221 *
222 * @param handle0 the HINTERNET connection to close
223 */
[492]224void InternetCloseHandle(HINTERNET* handle0){
225  int i=0;
226  for(i=0;i<handle0->nb;i++){
227    _HINTERNET handle=handle0->ihandle[i];
228    if(handle.hasCacheFile>0){
229      fclose(handle.file);
230      unlink(handle.filename);
231    }
232    else{
233      handle.pabyData = NULL;
234      handle.nDataAlloc = handle.nDataLen = 0;
235    }
236    if(handle0->ihandle[i].header!=NULL){
237      curl_slist_free_all(handle0->ihandle[i].header);
238      handle0->ihandle[i].header=NULL;
239    }
[621]240    if(handle.post!=NULL)
241      free(handle.post);
[492]242    free(handle.mimeType);
[534]243    handle.mimeType = NULL;
[1]244  }
[492]245  if(handle0->handle)
246    curl_multi_cleanup(handle0->handle);
247  free(handle0->agent);
248  for(i=handle0->nb-1;i>=0;i--){
249    free(handle0->waitingRequests[i]);
[1]250  }
251}
252
[579]253/**
254 * Create a new element in the download queue
255 *
256 * @param hInternet the HINTERNET connection to add the download link
257 * @param lpszUrl the url to download
258 * @param lpszHeaders the additional headers to be sent to the HTTP server
259 * @param dwHeadersLength the size of the additional headers
260 * @param dwFlags desired download mode (INTERNET_FLAG_NO_CACHE_WRITE for not using cache file)
261 * @param dwContext not used
262 */
[492]263HINTERNET InternetOpenUrl(HINTERNET* hInternet,LPCTSTR lpszUrl,LPCTSTR lpszHeaders,size_t dwHeadersLength,size_t dwFlags,size_t dwContext){
[1]264
265  char filename[255];
[490]266  struct MemoryStruct header;
267
[492]268  hInternet->ihandle[hInternet->nb].handle=curl_easy_init( );
269  hInternet->ihandle[hInternet->nb].hasCacheFile=0;
270  hInternet->ihandle[hInternet->nb].nDataAlloc = 0;
271  hInternet->ihandle[hInternet->nb].mimeType = NULL;
272  hInternet->ihandle[hInternet->nb].nDataLen = 0;
273  hInternet->ihandle[hInternet->nb].id = hInternet->nb;
274  hInternet->ihandle[hInternet->nb].nDataAlloc = 0;
275  hInternet->ihandle[hInternet->nb].pabyData = NULL;
[621]276  hInternet->ihandle[hInternet->nb].post = NULL;
[1]277
[492]278  curl_easy_setopt(hInternet->ihandle[hInternet->nb].handle, CURLOPT_COOKIEFILE, "ALL");
[490]279#ifndef TIGER
[492]280  curl_easy_setopt(hInternet->ihandle[hInternet->nb].handle, CURLOPT_COOKIELIST, "ALL");
[490]281#endif
[492]282  curl_easy_setopt(hInternet->ihandle[hInternet->nb].handle, CURLOPT_USERAGENT, hInternet->agent);
[490]283 
[492]284  curl_easy_setopt(hInternet->ihandle[hInternet->nb].handle,CURLOPT_FOLLOWLOCATION,1);
285  curl_easy_setopt(hInternet->ihandle[hInternet->nb].handle,CURLOPT_MAXREDIRS,3);
[490]286 
287  header.memory=NULL;
288  header.size = 0;
289
[492]290  curl_easy_setopt(hInternet->ihandle[hInternet->nb].handle, CURLOPT_HEADERFUNCTION, header_write_data);
291  curl_easy_setopt(hInternet->ihandle[hInternet->nb].handle, CURLOPT_WRITEHEADER, (void *)&header);
[490]292
293#ifdef MSG_LAF_VERBOSE
[492]294  curl_easy_setopt(hInternet->ihandle[hInternet->nb].handle, CURLOPT_VERBOSE, 1);
[490]295#endif
296
[1]297     
298  switch(dwFlags)
299    {
[492]300    case INTERNET_FLAG_NO_CACHE_WRITE:
301      hInternet->ihandle[hInternet->nb].hasCacheFile=-1;
302      curl_easy_setopt(hInternet->ihandle[hInternet->nb].handle, CURLOPT_WRITEFUNCTION, write_data_into);
303      curl_easy_setopt(hInternet->ihandle[hInternet->nb].handle, CURLOPT_WRITEDATA, (void*)&hInternet->ihandle[hInternet->nb]);
[1]304      break;
305    default:
[492]306      sprintf(hInternet->ihandle[hInternet->nb].filename,"/tmp/ZOO_Cache%d",(int)time(NULL));
307      hInternet->ihandle[hInternet->nb].filename[24]=0;
[1]308#ifdef MSG_LAF_VERBOSE
[492]309      fprintf(stderr,"file=%s",hInternet->ihandle[hInternet->nb].filename);
[1]310#endif
[492]311      hInternet->ihandle[hInternet->nb].filename=filename;
312      hInternet->ihandle[hInternet->nb].file=fopen(hInternet->ihandle[hInternet->nb].filename,"w+");
[1]313   
[492]314      hInternet->ihandle[hInternet->nb].hasCacheFile=1;
315      curl_easy_setopt(hInternet->ihandle[hInternet->nb].handle, CURLOPT_WRITEFUNCTION, NULL);
316      curl_easy_setopt(hInternet->ihandle[hInternet->nb].handle, CURLOPT_WRITEDATA, hInternet->ihandle[hInternet->nb].file);
317      hInternet->ihandle[hInternet->nb].nDataLen=0;
[1]318      break;
319    }
320#ifdef ULINET_DEBUG
321  fprintf(stderr,"URL (%s)\nBODY (%s)\n",lpszUrl,lpszHeaders);
322#endif
323  if(lpszHeaders!=NULL && strlen(lpszHeaders)>0){
324#ifdef MSG_LAF_VERBOSE
325    fprintf(stderr,"FROM ULINET !!");
[621]326    fprintf(stderr,"HEADER : [%s] %d\n",lpszHeaders,dwHeadersLength);
[1]327#endif
[492]328    curl_easy_setopt(hInternet->ihandle[hInternet->nb].handle,CURLOPT_POST,1);
[1]329#ifdef ULINET_DEBUG
330    fprintf(stderr,"** (%s) %d **\n",lpszHeaders,dwHeadersLength);
[492]331    curl_easy_setopt(hInternet->ihandle[hInternet->nb].handle,CURLOPT_VERBOSE,1);
[1]332#endif
[621]333    hInternet->ihandle[hInternet->nb].post=strdup(lpszHeaders);
334    curl_easy_setopt(hInternet->ihandle[hInternet->nb].handle,CURLOPT_POSTFIELDS,hInternet->ihandle[hInternet->nb].post);
335    curl_easy_setopt(hInternet->ihandle[hInternet->nb].handle,CURLOPT_POSTFIELDSIZE,(long)dwHeadersLength);
[1]336  }
[607]337  if(hInternet->ihandle[hInternet->nb].header!=NULL)
338    curl_easy_setopt(hInternet->ihandle[hInternet->nb].handle,CURLOPT_HTTPHEADER,hInternet->ihandle[hInternet->nb].header);
[1]339
[492]340  curl_easy_setopt(hInternet->ihandle[hInternet->nb].handle,CURLOPT_URL,lpszUrl);
[490]341
[492]342  curl_multi_add_handle(hInternet->handle,hInternet->ihandle[hInternet->nb].handle);
[490]343 
[621]344  hInternet->ihandle[hInternet->nb].header=NULL;
[492]345  ++hInternet->nb;
[490]346
[446]347#ifdef ULINET_DEBUG
348  fprintf(stderr,"DEBUG MIMETYPE: %s\n",hInternet.mimeType);
[478]349  fflush(stderr);
[446]350#endif
[492]351  return *hInternet;
[1]352};
353
[579]354/**
355 * Download all opened urls in the queue
356 *
357 * @param hInternet the HINTERNET structure containing the queue
358 * @return 0
359 */
[492]360int processDownloads(HINTERNET* hInternet){
361  int still_running=0;
362  int msgs_left=0;
363  int i=0;
364  do{
365    curl_multi_perform(hInternet->handle, &still_running);
366  }while(still_running); 
367  for(i=0;i<hInternet->nb;i++){
[534]368    char *tmp;
369    curl_easy_getinfo(hInternet->ihandle[i].handle,CURLINFO_CONTENT_TYPE,&tmp);
[539]370    if(tmp!=NULL)
371      hInternet->ihandle[i].mimeType=strdup(tmp);
[607]372    curl_easy_getinfo(hInternet->ihandle[i].handle,CURLINFO_RESPONSE_CODE,&hInternet->ihandle[i].code);
[492]373    curl_multi_remove_handle(hInternet->handle, hInternet->ihandle[i].handle);
374    curl_easy_cleanup(hInternet->ihandle[i].handle);
375  }
[509]376  return 0;
[492]377}
378
[579]379/**
380 * Initialize the CCookie for a specific index (hInternet.nb)
381 *
382 * @param hInternet the HINTERNET structure to know the CCookie index to reset
383 * @return 1
384 * @see HINTERNET
385 */
[1]386int freeCookieList(HINTERNET hInternet){
[492]387  memset(&CCookie[hInternet.nb][0],0,1024);
[1]388#ifndef TIGER
[492]389  curl_easy_setopt(hInternet.ihandle[hInternet.nb].handle, CURLOPT_COOKIELIST, "ALL");
[1]390#endif
391  return 1;
392}
393
[579]394/**
395 * Copy a downloaded content
396 *
397 * @param hInternet the _HINTERNET structure
398 * @param lpBuffer the memory space to copy the downloaded content
399 * @param dwNumberOfBytesToRead the size of lpBuffer
400 * @param lpdwNumberOfBytesRead number of bytes red
401 * @return 1 on success, 0 if failure
402 */
[492]403int InternetReadFile(_HINTERNET hInternet,LPVOID lpBuffer,int dwNumberOfBytesToRead, size_t *lpdwNumberOfBytesRead){
[1]404  int dwDataSize;
405
406  if(hInternet.hasCacheFile>0){
407    fseek (hInternet.file , 0 , SEEK_END);
408    dwDataSize=ftell(hInternet.file); //taille du ficher
409    rewind (hInternet.file);
410  }
411  else{
412    memset(lpBuffer,0,hInternet.nDataLen+1);
[375]413    memcpy(lpBuffer, hInternet.pabyData, hInternet.nDataLen );
[1]414    dwDataSize=hInternet.nDataLen;
415    free( hInternet.pabyData );
416    hInternet.pabyData=NULL;
417  }
418
419  if( dwNumberOfBytesToRead /* buffer size */ < dwDataSize )
420    return 0;
421
422#ifdef MSG_LAF_VERBOSE
423  printf("\nfile size : %dko\n",dwDataSize/1024);
424#endif
425
426  if(hInternet.hasCacheFile>0){
427    *lpdwNumberOfBytesRead = fread(lpBuffer,1,dwDataSize,hInternet.file); 
428  }
429  else{
430    *lpdwNumberOfBytesRead = hInternet.nDataLen;
431    free( hInternet.pabyData );
432    hInternet.pabyData = NULL;
433    hInternet.nDataAlloc = hInternet.nDataLen = 0;
434  }
435
[492]436  CCookie[hInternet.id][0]=0;
[1]437
438  if( *lpdwNumberOfBytesRead < dwDataSize )
439      return 0;
440  else
441      return 1; // TRUE
442}
443
[607]444
445/**
446 * Use basic authentication for accessing a ressource
447 *
448 * @param hInternet the _HINTERNET structure
449 * @param login the login to use to authenticate
450 * @param passwd the password to use to authenticate
451 */
452int setBasicAuth(HINTERNET hInternet,char* login,char* passwd){
453  char *tmp;
454  tmp=(char*)malloc((strlen(login)+strlen(passwd)+2)*sizeof(char));
455  sprintf(tmp,"%s:%s",login,passwd);
456  if(curl_easy_setopt(hInternet.ihandle[hInternet.nb].handle,CURLOPT_USERPWD,tmp)==CURLE_OUT_OF_MEMORY){
457    free(tmp);
458    return -1;
459  }
460  curl_easy_setopt(hInternet.ihandle[hInternet.nb].handle, CURLOPT_HTTPAUTH,CURLAUTH_ANY);
461  free(tmp);
462  return 0;
463}
Note: See TracBrowser for help on using the repository browser.

Search

ZOO Sponsors

http://www.zoo-project.org/trac/chrome/site/img/geolabs-logo.pnghttp://www.zoo-project.org/trac/chrome/site/img/neogeo-logo.png http://www.zoo-project.org/trac/chrome/site/img/apptech-logo.png http://www.zoo-project.org/trac/chrome/site/img/3liz-logo.png http://www.zoo-project.org/trac/chrome/site/img/gateway-logo.png

Become a sponsor !

Knowledge partners

http://www.zoo-project.org/trac/chrome/site/img/ocu-logo.png http://www.zoo-project.org/trac/chrome/site/img/gucas-logo.png http://www.zoo-project.org/trac/chrome/site/img/polimi-logo.png http://www.zoo-project.org/trac/chrome/site/img/fem-logo.png http://www.zoo-project.org/trac/chrome/site/img/supsi-logo.png http://www.zoo-project.org/trac/chrome/site/img/cumtb-logo.png

Become a knowledge partner

Related links

http://zoo-project.org/img/ogclogo.png http://zoo-project.org/img/osgeologo.png