Ignore:
Timestamp:
Feb 1, 2018, 11:27:28 AM (6 years ago)
Author:
djay
Message:

Add the capability to publish heatmap or any templated mapfile using the epecific msInclude and msLayer keys for an output. For MapServer? published output, define 4096 as the default maxsize and use pixel width or height for raster files. use the correct MapServer? imagemode depending on GDALGetRasterDataType (MS_IMAGEMODE_BYTE for GDT_Byte, MS_IMAGEMODE_INT16 for GDT_Int16 and MS_IMAGEMODE_FLOAT32 for GDT_Float32). Create a text file (.maps) listing every mapfiles created for a MapServer? published output (or inputs) using saveMapNames function. Fixes in ulinet, use uuid for naming temporary files. Add dialect input to the ogr2ogr service. Use the .maps file for removing a file from the DeleteData? service

File:
1 edited

Legend:

Unmodified
Added
Removed

  • branches/prototype-v0/zoo-project/zoo-kernel/service_internal_hpc.c

    r860 r862  
    6060  while(cur!=NULL && cur->defaults!=NULL){
    6161    map* mimeType=getMap(cur->defaults->content,"mimeType");
    62     if(mimeType!=NULL){
     62    map* useMS=getMap(cur->defaults->content,"useMapserver");
     63    if(mimeType!=NULL && useMS!=NULL && strncasecmp(useMS->value,"true",4)==0){
    6364      int geo=isGeographic(mimeType->value);
    6465      if(geo>0){
    … …  
    159160        free(tmp[0]);
    160161        //addToMap(cur->content,"internal","true");
    161       }
     162      }     
    162163    }
    163164    cur=cur->next;
    … …  
    332333  int parameters_cnt=0;
    333334  while(input!=NULL && input->content!=NULL){
    334     if(getMaps(*real_outputs,input->name)==NULL){
     335    map* isInRequest=getMap(input->content,"inRequest");
     336    map* minNb=getMap(input->content,"minoccurs");
     337    if(getMaps(*real_outputs,input->name)==NULL &&
     338       ( (isInRequest!=NULL && strncasecmp(isInRequest->value,"true",4)==0)
     339         || (minNb!=NULL && atoi(minNb->value)>0) ) ){
    335340      parameters_cnt+=1;
    336341      if(parameters_cnt==1)
    … …  
    345350          dumpMapsValuesToFiles(main_conf,&input);
    346351          addToMap(input->content,"toPublish","true");
    347           addToMap(input->content,"useMapserver","true");
     352          //addToMap(input->content,"useMapserver","true");
    348353        }
    349354        if(getMap(input->content,"cache_file")!=NULL){
    … …  
    362367            setMapArray(input->content,"targetPath",i,targetPath);
    363368            setMapArray(input->content,"localPath",i,tmp->value);
    364             addToUploadQueue(main_conf,input);
     369            map* tmp1=getMapArray(input->content,"value",i);
     370            if(tmp1!=NULL){
     371              free(tmp1->value);
     372              tmp1->value=strdup("empty");
     373            }
    365374            if(i==0){
    366375              parameters[parameters_cnt-1]=(char*)malloc((strlen(input->name)+strlen(targetPath)+3)*sizeof(char));
    367376              sprintf(parameters[parameters_cnt-1],"-%s %s",input->name,targetPath);
    368377            }else{
     378              fprintf(stderr,"%s %d\n",__FILE__,__LINE__);
     379              fflush(stderr);
    369380              char *tmpStr=zStrdup(parameters[parameters_cnt-1]);
    370381              parameters[parameters_cnt-1]=(char*)realloc(parameters[parameters_cnt-1],(strlen(tmpStr)+strlen(targetPath)+2)*sizeof(char));
    371382              sprintf(parameters[parameters_cnt-1],"%s %s",tmpStr,targetPath);
    372383              free(tmpStr);
     384              fprintf(stderr,"%s %d\n",__FILE__,__LINE__);
     385              fflush(stderr);
    373386            }
    374387            free(targetPath);
    375388          }
     389          addToUploadQueue(main_conf,input);
    376390        }else{
    377391          // ???
    … …  
    396410#endif
    397411  invokeCallback(m,inputs,NULL,1,1);
     412  invokeCallback(m,inputs,NULL,2,0);
    398413  if(getMapFromMaps(m,"lenv","mapError")!=NULL){
    399     invokeCallback(m,inputs,NULL,7,0);
     414    invokeCallback(*main_conf,inputs,NULL,7,0);
    400415    return -1;
    401416  }
    402   invokeCallback(m,inputs,NULL,2,0);
    403417#ifdef HPC_DEBUG
    404418  fprintf(stderr,"************************* %s %d \n\n",__FILE__,__LINE__);
    … …  
    408422  // Upload data on HPC
    409423  if(runUpload(main_conf)==false){
    410     errorException (m, _("Unable to lock the file for upload!"),
     424    errorException (*main_conf, _("Unable to lock the file for upload!"),
    411425                    "InternalError", NULL);
    412426#ifdef HPC_DEBUG
    413427    fprintf(stderr,"************************* %s %d \n\n",__FILE__,__LINE__);
    414428#endif
    415     invokeCallback(m,inputs,NULL,7,0);
     429    invokeCallback(*main_conf,inputs,NULL,7,0);
    416430#ifdef HPC_DEBUG
    417431    fprintf(stderr,"************************* %s %d \n\n",__FILE__,__LINE__);
    … …  
    429443  // Add the filename to generate for every output to parameters
    430444  input=*real_outputs;
    431   // TODO: fix appendOutputParameters
    432   //appendOutputParameters(input,parameters,&parameters_cnt,s,uuid,targetPathMap);
    433445#ifdef HPC_DEBUG
    434446  dumpMaps(input);
    435447#endif
    436448  while(input!=NULL){
    437     // TODO: parse all outputs including inner outputs if required.
     449    // Parse all outputs including inner outputs if required.
    438450    if(input->child==NULL){
    439451      // Name every files that should be produced by the service execution
    … …  
    453465      char *targetPath=(char*)malloc((strlen(targetPathMap->value)+strlen(targetName)+2)*sizeof(char));
    454466      sprintf(targetPath,"%s/%s",targetPathMap->value,targetName);
     467      map *tmpUrl=getMapFromMaps(*main_conf,"main","tmpUrl");
     468      char *targetUrl=(char*)malloc((strlen(tmpUrl->value)+strlen(targetName)+2)*sizeof(char));
     469      sprintf(targetUrl,"%s/%s",tmpUrl->value,targetName);
    455470      free(targetName);
    456471      setMapInMaps(*real_outputs,input->name,"generated_file",targetPath);
    457       if(strcasecmp(input->name,"wms_link")!=0&&
    458          strcasecmp(input->name,"wcs_link")!=0 &&
    459          strcasecmp(input->name,"wfs_link")!=0){
     472      addToMap(input->content,"generated_url",targetUrl);
     473      free(targetUrl);
     474      {
    460475        parameters_cnt+=1;
    461476        if(parameters_cnt==1)
    … …  
    494509      char *targetPath=(char*)malloc((strlen(targetPathMap->value)+strlen(targetName)+2)*sizeof(char));
    495510      sprintf(targetPath,"%s/%s",targetPathMap->value,targetName);
     511      map *tmpUrl=getMapFromMaps(*main_conf,"main","tmpUrl");
     512      char *targetUrl=(char*)malloc((strlen(tmpUrl->value)+strlen(targetName)+2)*sizeof(char));
     513      sprintf(targetUrl,"%s/%s",tmpUrl->value,targetName);
    496514      free(targetName);
    497515      addToMap(input->content,"generated_file",targetPath);
    498516      addToMap(input->content,"storage",targetPath);
     517      addToMap(input->content,"generated_url",targetUrl);
     518      free(targetUrl);
    499519      if(strcasecmp(input->name,"wms_link")!=0&&
    500520         strcasecmp(input->name,"wcs_link")!=0 &&
    … …  
    623643    setMapInMaps(*main_conf,"lenv","message",_("There is no remote_work_path defined in your section!"));
    624644    setMapInMaps(*main_conf,"lenv","status","failed");
    625     errorException (m, _("There is no remote_work_path defined in your section!"),
     645    errorException (*main_conf, _("There is no remote_work_path defined in your section!"),
    626646                    "InternalError", NULL);
    627647#ifdef HPC_DEBUG
    … …  
    629649    fflush(stderr);
    630650#endif
    631     invokeCallback(m,NULL,NULL,7,0);
     651    invokeCallback(*main_conf,NULL,NULL,7,0);
    632652#ifdef HPC_DEBUG
    633653    fprintf(stderr,"************************* %s %d \n\n",__FILE__,__LINE__);
    … …  
    645665  free(scriptPath);
    646666  if(copy0!=true){
    647     setMapInMaps(m,"lenv","message",_("Unable to upload the script"));
    648     invokeCallback(m,NULL,NULL,7,0);
    649     errorException(m,_("Unable to upload the script"),"NoApplicableCode",NULL);
     667    setMapInMaps(*main_conf,"lenv","message",_("Unable to upload the script"));
     668    invokeCallback(*main_conf,NULL,NULL,7,0);
     669    errorException(*main_conf,_("Unable to upload the script"),"NoApplicableCode",NULL);
    650670    return -1;
    651671  }
    … …  
    688708    fflush(stderr);
    689709#endif
    690     invokeCallback(m,NULL,NULL,7,0);
     710    invokeCallback(*main_conf,NULL,NULL,7,0);
    691711#ifdef HPC_DEBUG
    692712    fprintf(stderr,"************************* %s %d \n\n",__FILE__,__LINE__);
    … …  
    694714#endif
    695715    sprintf(tmpS, "Cannot execute the HPC ZOO-Service %s using %s: %s", s->name, configurationId, tmpPath->value);
    696     errorException(m,tmpS,"NoApplicableCode",NULL);
     716    errorException(*main_conf,tmpS,"NoApplicableCode",NULL);
    697717    free(command);
    698718    free(targetPath);
    699719    ssh_close(*main_conf);
    700720    removeReadLocks(main_conf);
    701     sleep(1);
    702721    return -1;
    703722  }
    … …  
    728747  if (bind(fd, (struct sockaddr*)&addr, sizeof(addr)) == -1) {
    729748    perror("bind error");
    730     setMapInMaps(m,"lenv","message",_("Unable to bind socket!"));
    731     errorException (m, _("Unable to bind socket!"),
     749    setMapInMaps(*main_conf,"lenv","message",_("Unable to bind socket!"));
     750    errorException (*main_conf, _("Unable to bind socket!"),
    732751                    "InternalError", NULL);
    733752#ifdef HPC_DEBUG
    … …  
    735754    fflush(stderr);
    736755#endif
    737     invokeCallback(m,NULL,NULL,7,0);
     756    invokeCallback(*main_conf,NULL,NULL,7,0);
    738757    removeReadLocks(main_conf);
    739758#ifdef HPC_DEBUG
    … …  
    749768  if (listen(fd, 5) == -1) {
    750769    setMapInMaps(*main_conf,"lenv","message",_("Listen error"));
    751     errorException (m, _("Listen error"),
     770    errorException (*main_conf, _("Listen error"),
    752771                    "InternalError", NULL);
    753772#ifdef HPC_DEBUG
    … …  
    755774    fflush(stderr);
    756775#endif
    757     invokeCallback(m,NULL,NULL,7,0);
     776    invokeCallback(*main_conf,NULL,NULL,7,0);
    758777    removeReadLocks(main_conf);
    759778#ifdef HPC_DEBUG
    … …  
    765784  if ( (cl = accept(fd, NULL, NULL)) == -1) {
    766785    setMapInMaps(*main_conf,"lenv","message",_("Accept error"));
    767     errorException (m, _("Accept error"),
     786    errorException (*main_conf, _("Accept error"),
    768787                    "InternalError", NULL);
    769788#ifdef HPC_DEBUG
    … …  
    771790    fflush(stderr);
    772791#endif
    773     invokeCallback(m,NULL,NULL,7,0);
     792    invokeCallback(*main_conf,NULL,NULL,7,0);
    774793    removeReadLocks(main_conf);
    775794#ifdef HPC_DEBUG
    … …  
    788807    while ( (rc=read(cl,buf,10)) ) {     
    789808      if(rc==0){
    790         sleep(1);
    791809        setMapInMaps(*main_conf,"lenv","message",_("Read closed"));
    792         errorException (m, _("Read closed"),
     810        errorException (*main_conf, _("Read closed"),
    793811                        "InternalError", NULL);
    794812#ifdef HPC_DEBUG
    … …  
    796814        fflush(stderr);
    797815#endif
    798         invokeCallback(m,NULL,NULL,7,0);
     816        invokeCallback(*main_conf,NULL,NULL,7,0);
    799817        removeReadLocks(main_conf);
    800818#ifdef HPC_DEBUG
    … …  
    806824        if(rc<0){
    807825          setMapInMaps(*main_conf,"lenv","message",_("Read error"));
    808           errorException (m, _("Read error"),
     826          errorException (*main_conf, _("Read error"),
    809827                          "InternalError", NULL);
    810828#ifdef HPC_DEBUG
    … …  
    812830          fflush(stderr);
    813831#endif
    814           invokeCallback(m,NULL,NULL,7,0);
     832          invokeCallback(*main_conf,NULL,NULL,7,0);
    815833          removeReadLocks(main_conf);
    816834#ifdef HPC_DEBUG
    … …  
    872890                free(targetPath);
    873891              }else{
    874                 char *tmpStr=(char*)malloc((strlen(filename)+strlen(_("Unable to fetch the remote file for %s"))+1)*sizeof(char));
    875                 sprintf(tmpStr,_("Unable to fetch the remote file for %s"),filename);
    876                 setMapInMaps(m,"lenv","message",tmpStr);
    877                 free(tmpStr);
    878                 invokeCallback(m,NULL,NULL,7,0);
     892                map* hpcStdErr=getMapFromMaps(*main_conf,"henv","StdErr");
     893                if(hpcStdErr!=NULL && ssh_fetch(*main_conf,targetPath,hpcStdErr->value,ssh_get_cnt(m))==0){
     894                  struct stat f_status;
     895                  int ts=stat(targetPath, &f_status);
     896                  if(ts==0) {
     897                    char* fcontent = NULL;
     898                    fcontent=(char*)malloc(sizeof(char)*(f_status.st_size+1));
     899                    FILE* f=fopen(targetPath,"rb");
     900                    fread(fcontent,f_status.st_size,1,f);
     901                    int fsize=f_status.st_size;
     902                    fcontent[fsize]=0;
     903                    fclose(f);
     904                    setMapInMaps(*main_conf,"lenv","message",fcontent);
     905                    free(fcontent);
     906                  }else{
     907                    char *tmpStr=(char*)malloc((strlen(filename)+strlen(_("Unable to fetch the remote file for %s"))+1)*sizeof(char));
     908                    sprintf(tmpStr,_("Unable to fetch the remote file for %s"),filename);
     909                    setMapInMaps(*main_conf,"lenv","message",tmpStr);
     910                    free(tmpStr);
     911                  }
     912                }else{
     913                  char *tmpStr=(char*)malloc((strlen(filename)+strlen(_("Unable to fetch the remote file for %s"))+1)*sizeof(char));
     914                  sprintf(tmpStr,_("Unable to fetch the remote file for %s"),filename);
     915                  setMapInMaps(*main_conf,"lenv","message",tmpStr);
     916                  free(tmpStr);
     917                }
     918                invokeCallback(*main_conf,NULL,NULL,7,0);
    879919                return SERVICE_FAILED;
    880920              }
    … …  
    882922          }else{
    883923            map* generatedFile=getMap(input->content,"generated_file");
     924            map* generatedUrl=getMap(input->content,"generated_url");
    884925            if(generatedFile!=NULL){
    885926              char* filename=strrchr(generatedFile->value,'/');
    … …  
    895936                maps* output=getMaps(*real_outputs,input->name);
    896937                setMapInMaps(output->child,"download_link","generated_file",targetPath);
     938                setMapInMaps(output->child,"download_link","generated_url",generatedUrl->value);
    897939                setMapInMaps(output->child,"download_link","storage",targetPath);
    898940                setMapInMaps(output->child,"download_link","useMapserver","false");
    … …  
    908950                if(getMaps(output->child,"wcs_link")!=NULL){
    909951                  sprintf(serviceName,"wcs_link");
    910                   setMapInMaps(output->child,serviceName,"msOgc","WCS");
    911                   setMapInMaps(output->child,serviceName,"requestedMimeType","image/tiff");
    912                 }
    913                 else{
     952                  setMapInMaps(output->child,"wcs_link","msOgc","WCS");
     953                }else{
    914954                  sprintf(serviceName,"wfs_link");
    915                   setMapInMaps(output->child,serviceName,"msOgc","WFS");
    916                   setMapInMaps(output->child,serviceName,"requestedMimeType","text/xml");
    917                 }
     955                  setMapInMaps(output->child,"wfs_link","msOgc","WFS");
     956                }
    918957                setMapInMaps(output->child,serviceName,"storage",targetPath);
    919958                setMapInMaps(output->child,serviceName,"generated_file",targetPath);
    … …  
    958997      }else{
    959998        // Try to access remotely to the log file and return a more relevant error message
    960         setMapInMaps(m,"lenv","message",_("HPC Execution failed!"));
    961         errorException (m, _("HPC Execution failed!"),
     999        setMapInMaps(*main_conf,"lenv","message",_("HPC Execution failed!"));
     1000        errorException (*main_conf, _("HPC Execution failed!"),
    9621001                        "InternalError", NULL);
    9631002#ifdef HPC_DEBUG
    … …  
    9651004        fflush(stderr);
    9661005#endif
    967         invokeCallback(m,NULL,NULL,7,0);
     1006        invokeCallback(*main_conf,NULL,NULL,7,0);
    9681007#ifdef HPC_DEBUG
    9691008        fprintf(stderr,"************************* %s %d \n\n",__FILE__,__LINE__);
    … …  
    9711010#endif
    9721011      }
    973       //free(buf);
    9741012    }
    9751013    if(hasPassed<0){
    9761014      perror("Failed to read");
    9771015      setMapInMaps(*main_conf,"lenv","message",_("Unable to parse the value returned by remote execution"));
    978       errorException (m, _("Unable to parse the value returned by remote execution"),
     1016      errorException (*main_conf, _("Unable to parse the value returned by remote execution"),
    9791017                      "InternalError", NULL);
    9801018#ifdef HPC_DEBUG
    … …  
    9821020      fflush(stderr);
    9831021#endif
    984       invokeCallback(m,NULL,NULL,7,0);
     1022      invokeCallback(*main_conf,NULL,NULL,7,0);
    9851023#ifdef HPC_DEBUG
    9861024      fprintf(stderr,"************************* %s %d \n\n",__FILE__,__LINE__);
Note: See TracChangeset for help on using the changeset viewer.

Download now !

Search

Context Navigation

Main Navigation

ZOO Sponsors

http://www.zoo-project.org/trac/chrome/site/img/geolabs-logo.pnghttp://www.zoo-project.org/trac/chrome/site/img/neogeo-logo.png http://www.zoo-project.org/trac/chrome/site/img/apptech-logo.png http://www.zoo-project.org/trac/chrome/site/img/3liz-logo.png http://www.zoo-project.org/trac/chrome/site/img/gateway-logo.png

Become a sponsor !

Knowledge partners

http://www.zoo-project.org/trac/chrome/site/img/ocu-logo.png http://www.zoo-project.org/trac/chrome/site/img/gucas-logo.png http://www.zoo-project.org/trac/chrome/site/img/polimi-logo.png http://www.zoo-project.org/trac/chrome/site/img/fem-logo.png http://www.zoo-project.org/trac/chrome/site/img/supsi-logo.png http://www.zoo-project.org/trac/chrome/site/img/cumtb-logo.png

Become a knowledge partner

Related links

http://zoo-project.org/img/ogclogo.png http://zoo-project.org/img/osgeologo.png