#!/usr/bin/env perl

=pod

=head1 NAME

download-organism

=head1 VERSION

$program_version

=head1 DESCRIPTION

Download organism(s) from a RSAT server or genome repository in order to support it on
this RSAT instance.

Organisms can be loaded one by one (option -org), or by taxon (option -taxon).

=head1 AUTHORS

Jacques.van.Helden@ulb.ac.be, bcontreras@eead.csic.es

=head1 CATEGORY

Data management

=head1 REQUIREMENTS

This program relies on I<wget> to download genomes from a remote RSAT server/genome repo to the local machine.

=head1 USAGE

To list all remote servers/genome repositories and see the default one:

 download-organism -servers

To list all organisms available in remote servers/genome repositories:

 download-organism -listall

Download one or several organisms by species identifiers.

 download-organism -org Species_id_1 -org Species_id_2 ...

Download all the organisms belonging to a given taxon.

 download-organism -taxon taxon

=head1 OUTPUT

By default, the downloaded organisms are stored in the $RSAT genome
repository ($RSAT/public_html/data/genomes).

=head1 SEE ALSO

=head2 suported-organisms-server

The program I<supported-organisms-server> can be used to obtain the list of
supported organisms on remote RSAT servers or repositories.

=head2 install-organism

The program I<install-organism> performs all the formatting and
calibration tasks for importing genomes from the reference databases
(NCBI, Ensembl) to RSAT. 

This differs from I<download-organism>, which downloads & copies the
RSAT-formatted genomes from a RSAT server.

If a genome is available on the RSAT server, it is recommended to use
download-genomes in order to obtain it immediately in the RSAT format,
rather than install-genomes.

=head1 WISH LIST

=over


=item B<-taxfreq>

Download the taxon-wise oligo and dyad frequencies. Those are required for
comparative genomics approaches (footprint-discovery, footprint-scan).

=back

=cut


BEGIN {
  if ($0 =~ /([^(\/)]+)$/) {
    push (@INC, "$`lib/");
  }
}
require "RSA.lib";
require RSAT::server;
require RSAT::util;

################################################################
## Main package
package main;
{

  ################################################################
  ## Initialise parameters
  my $start_time = &RSAT::util::StartScript();
  my $get_command = "wget";
  my $program_version = 2.0; # no WS

  ## Default action is to download. This variable is used to display status
  my $action = "Downloading";

  my $verbose = 0;
  my $dry = 0; ## Print the commands without executing them
  my $die_on_error = 0;
  our $listall = 0;
  our $servers = 0;

  my $def_genomes_path = 'data/genomes/';

  # remote servers/genome repos, 
  # require $def_supported_orgs_file in place for taxon operations
  our @rsat_repos = ();
  our @rsat_servers = ();
 
  ## List of organisms to install
  @organisms = ();

  ## List of taxa to install
  @taxa = ();

  ## Local genomes directory
  $local_genomes_dir = $ENV{RSAT}."/public_html/data/genomes/";

  ## Indexes for organism attributes
  %last_update = ();
  %source = ();

  ## First and lst organisms
  our $skip_org = 0;
  our $last_org = 0;
  our $no_blast = 1;

  ################################################################
  ## Read argument values
  ReadArguments();

  ################################################################
  ## Check that the web aspirator is working
  $which_aspirator = &RSAT::server::GetProgramPath($get_command);
  RSAT::message::Debug($which_aspirator) if ($main::verbose >= 4);
  unless ($which_aspirator) {
    RSAT::error::FatalError("The program ".$get_command.
      " is not found in your path. This program is required for ".
      $action." data from the server. ");
  }

  ################################################################
  ## Check argument values
  if ((scalar(@organisms) == 0) && (scalar(@taxa) == 0) && 
    ($orglist_file eq "") && $listall == 0 && $servers ==0) {
    FatalError("You must specify at least -org, -taxon, -org_list, -servers or -listall.");
  }

  ## Output directory
  RSAT::message::Info("Local genome directory", $local_genomes_dir) if ($main::verbose >= 2);
  RSAT::util::CheckOutDir($local_genomes_dir);

  ################################################################
  ## Open output stream
  $main::out = OpenOutputFile($main::outfile{output});

  ################################################################
  ## Print verbose
  Verbose() if ($main::verbose);

  ################################################################
  ## Get list of remote servers and genome repos
  my ($server, $type);
  $cmd = RSAT::server::GetProgramPath("supported-organisms-server");
  $cmd .= " -servers";
  RSAT::message::TimeWarn("Getting list of remote servers\n; ", $cmd) if ($main::verbose >= 2);
  open(SUPORGS,"$cmd |") ||
     RSAT::error::FatalError("Failed $cmd");
  while(<SUPORGS>) {
    if(/^(\S+)\t#(\S+)/) {
      ($server, $type) = ($1, $2);

      print $main::out "$server\t#$type\n" if ($servers == 1);

      if($type eq 'repo') {
        push(@rsat_repos, $server);
      } else {
        push(@rsat_servers, $server);
      }
    } 
  }
  close(SUPORGS);

  ## Set default server/repo
  if(scalar(@rsat_repos)) {
    $RSAT_SERVER=$rsat_repos[0]
  } elsif(scalar(@rsat_servers)) {
    $RSAT_SERVER=$rsat_servers[0]
  } else {
    RSAT::error::FatalError("Failed parsing remote servers/repos");
  }

  RSAT::message::Info("default server: $server");

  exit if($servers == 1);

  ################################################################
  ## Get list of all organisms available in remote servers and genome repos
  RSAT::message::TimeWarn("Listing all orgs available in remote servers/repos") if ($main::verbose >= 2);
  foreach $server (@rsat_repos,@rsat_servers) {
    $cmd = RSAT::server::GetProgramPath("supported-organisms-server");
    $cmd .= " -url $server -return ID,source,taxonomy";
    RSAT::message::TimeWarn("Querying remote server\n; ", $cmd) if ($main::verbose >= 2);
    open(SUPORGS,"$cmd |") ||
     RSAT::error::FatalError("Failed $cmd");
    while(<SUPORGS>) {
      print $main::out "$server\t$_";
    }
    close(SUPORGS);
  }

  exit if($listall == 1);


  ################################################################
  ## Obtain the lists of organisms for selected taxa
  foreach my $taxon (@taxa) {
    RSAT::message::TimeWarn("Getting list of organisms from RSAT server for taxon", $taxon);
    my $orgs_for_taxa;
    RSAT::message::Info("Getting list of organisms to download for taxon", $taxon);
    $cmd = &RSAT::server::GetProgramPath("supported-organisms-server");
    $cmd .= " -url ".$RSAT_SERVER." -return ID -taxon ".$taxon." | grep -v '^#'";
    
    RSAT::message::TimeWarn("\n; ", $cmd) if ($main::verbose >= 3);
    $orgs_for_taxa = `$cmd`;

    chomp ($orgs_for_taxa);
    @orgs_for_taxa = sort(split(/\s+/, $orgs_for_taxa));
    push @organisms, @orgs_for_taxa;
  } 

  ################################################################
  ## Read list of organisms from a file
  if ($main::orglist_file) {
      RSAT::message::Info("Reading list of organisms form file", $main::orglist_file ) if ($main::verbose >= 1);
      my @orgs_from_file = ();
      ($main::orglist) = OpenInputFile($main::orglist_file);
      while (<$main::orglist>) {
	  chomp();
	  s/\r/\n/g;	  ## Suppress Windows-specific carriage return
	  next if /^;/;		## Comment line
	  next if /^\#/;		## Header line
	  next if /^\--/;		## SQL comment line
	  next unless /\S/;		## Empty line
	  my ($org) = split /\s/;
	  $org = trim($org); ## Remove leading and trailing spaces
	  push @orgs_from_file, $org;
      }
      close $main::orglist;
      if (scalar(@orgs_from_file) == 0) {
	RSAT::error::FatalError("The organism file", $main::orglist_file, 
          "should contain at least one valid organism name.\n",
          "Use the command supported-organisms-server to obtain the list of organisms supported on the server.");
      } else {
	RSAT::message::Info(scalar(@orgs_from_file), "organisms in file", 
          $main::orglist_file) if ($main::verbose >= 1);
	push @organisms, @orgs_from_file;
      }
  }
  
  &RSAT::message::Info($action, scalar(@organisms), "organisms");
  my $i = 0;
  foreach my $org (@organisms) {
    $i++;
    warn join ("\t", ";", $i, $org) , "\n";
  }

  ################################################################
  ## Get the update dates + source from the server (in order to restrict the
  ## number of queries, get dates for all organisms and store them for
  ## the selected organisms only)
  RSAT::message::TimeWarn("Getting information about supported organisms from server") if ($main::verbose >= 2);


  ################################################################
  ## Get organism-specific parameters (data source, last update date).
  my @all_organisms;
  $cmd = &RSAT::server::GetProgramPath("supported-organisms-server");
  $cmd .= " -url ".$RSAT_SERVER." -v 0 -return ID,last_update,source | grep -v '^#'";
  
  RSAT::message::TimeWarn("Getting list of supported organisms from remote server\n; ", $cmd) if ($main::verbose >= 2);
  @all_organisms = `$cmd`;

  &RSAT::message::Info(scalar(@all_organisms), "supported organisms on server", $RSAT_SERVER) if ($main::verbose >= 2);

  foreach my $org_line (@all_organisms) {
    next unless ($org_line); ## Skip empty line returned by RSAT Web service
    chomp($org_line);
    my ($org, $date, $source) = split /\t/, $org_line;
    if ($org) {
      $last_update{$org} = $date || '<NA>';
      $source{$org} = $source || '<NA>';
    }
    RSAT::message::Debug("Server supported", $org_line, $org, $last_update{$org}, $source{$org}) if ($main::verbose >= 5);
  } 

  ################################################################
  ## Download the genomes
  my $org_nb = scalar(@organisms);
  $i = 0;
  
  ## Iterate over organisms
  foreach my $org (@organisms) {
    $i++;

    next if (($skip_org > 0) && ($i <= $skip_org));
    next if (($last_org > 0) && ($i > $last_org));

    ## Transfer (download or upload) the organism data
    unless ($no_data) {

	## Create destination directory
	$org_dir = $local_genomes_dir."/".$org;
	RSAT::util::CheckOutDir($org_dir);

	## download the genome
	$cmd = $which_aspirator;
	if ($get_command eq "wget") {
	    my $RSAT_SERVER_ROOT = $RSAT_SERVER;
	    $RSAT_SERVER_ROOT =~ s|http://||;
	    if ($RSAT_SERVER_ROOT =~ /\//) {
		$RSAT_SERVER_ROOT = '/'.$'; #'
	    }

	    RSAT::message::TimeWarn($action, "genome", $i."/".$org_nb, $org, 
				     "RSAT server", $RSAT_SERVER) if ($main::verbose >= 1);
	    RSAT::message::Debug("RSAT server root", $RSAT_SERVER_ROOT) if ($main::verbose >= 3);
	    $cmd .= " --reject jobs --reject 'index.html*'";
            
	    if(grep(/$RSAT_SERVER/,@main::rsat_servers)) {
              $cmd .= " -X ${RSAT_SERVER_ROOT}/$def_genomes_path/$org/blast_hits" if ($no_blast);
            } else {
              $cmd .= " -X ${RSAT_SERVER_ROOT}/$org/blast_hits" if ($no_blast);
	    }

	    if ($main::verbose <= 3) {
		## quiet mode for wget
		$cmd .= " --quiet"; 
	    } elsif ($main::verbose <= 4) {
		## "low-verbosity" mode for wget (the default mode is VERY
		## verbosy, the "low-verbosity" mode is still quite verbosy)
		$cmd .= " -nv"; 
	    } else {
		## "verbosity" mode for wget (VERY verbosy)
		$cmd .= " -v"; 
	    }
	    $cmd .= " --passive-ftp";
	    $cmd .= " --no-parent";
	    $cmd .= " --recursive";
	    $cmd .= " --timestamping";
	    $cmd .= " --relative";
	    $cmd .= " --dont-remove-listing";
	    $cmd .= " --convert-links";
	    $cmd .= " --exclude-directories=jobs";
	    $cmd .= " --directory-prefix=".$local_genomes_dir."/";

            if(grep(/$RSAT_SERVER/,@main::rsat_servers)) {
              $cmd .= " --no-host-directories --cut-dirs=3 ${RSAT_SERVER}/$def_genomes_path/$org/";
            } else { 
              $cmd .= " --no-host-directories --cut-dirs=2 ${RSAT_SERVER}/$org/";
	    }
	    
	    #$cmd =~ s|//|/|g;
	    #die "\n", $cmd, "\n";
	} 
	
	doit($cmd, $dry, $die_on_error, $verbose, 0, 0);
    }


    ## Update organism configuration in the RSAT target server
    unless ($no_config) {
      my $source = $source{$org} || '<NA>';
      my $date = $last_update{$org} || '<NA>';
      RSAT::message::TimeWarn("Updating organism",$org) if ($main::verbose >= 2);
      $cmd = "install-organism";
      $cmd .= " -v 1 -task config,phylogeny -org ".$org;
      $cmd .= " -date \"".$date."\"";
      $cmd .= " -source \"".$source."\"";
#    $cmd .= " >>& install-organisms_log.txt";
      if ($main::upload) {
	$cmd = "ssh $ssh_login '".$source_bashrc_cmd."$cmd'";
      }
      doit($cmd, $dry, $die_on_error, $verbose, 0, 0);
      RSAT::message::TimeWarn("Updating config and phylogeny\n", $cmd) if ($main::verbose >= 2);
    }
  }

  my $exec_time = RSAT::util::ReportExecutionTime($start_time);
  print $main::out $exec_time if ($main::verbose >= 1);
  exit(0);
}

################################################################
################### SUBROUTINE DEFINITION ######################
################################################################


################################################################
## Display full help message 
sub PrintHelp {
    system "pod2text -c $0";
    exit()
}

################################################################
## Display short help message
sub PrintOptions {
    &PrintHelp();
}

################################################################
## Read arguments 
sub ReadArguments {
    my $arg;
    my @arguments = @ARGV; ## create a copy to shift, because we need ARGV to report command line in &Verbose()
    while (scalar(@arguments) >= 1) {
      $arg = shift (@arguments);
	## Verbosity
=pod

=head1 OPTIONS

=over 4

=item B<-v #>

Level of verbosity (detail in the warning messages during execution)

=cut
	if ($arg eq "-v") {
	    if (&IsNatural($arguments[0])) {
		$main::verbose = shift(@arguments);
	    } else {
		$main::verbose = 1;
	    }

	    ## Help message
=pod

=item B<-h>

Display full help message

=cut
	} elsif ($arg eq "-h") {
	    &PrintHelp();

	    ## List of options
=pod

=item B<-help>

Same as -h

=cut
	} elsif ($arg eq "-help") {
	    &PrintOptions();

	    ## Selected organism(s)
=pod

=item B<-org organism>

ID of the organism to be downloaded. This ID is generally the full
name of the organism, where spaces and weird characters have been
replaced by an underscore (_). You can get IDs with help from
supported-organisms-server .

Ex: download-organism -v 1 -org Escherichia_coli_str._K-12_substr._MG1655_GCF_000005845.2_ASM584v2 

The option -org can be used iteratively on the command line to specify
multiple organisms.

 download-organism -v 1 -org Escherichia_coli_str._K-12_substr._MG1655_GCF_000005845.2_ASM584v2 \
    -org Saccharomyces_cerevisiae

=cut

} elsif ($arg eq "-org") {
	  push @organisms,  shift(@arguments);

=pod

=item B<-org_list>

This option gives the posibility to specify a set of genome IDs rather than a taxon. 
Orthologs will only be searched in the organisms belonging to the given list.

File format: each row should contain the identifier of one
organism. Lines starting with a semicolumn are ignored.

=cut

} elsif ($arg eq "-org_list") {
    $main::orglist_file= shift(@arguments);

	    ## Selected taxon (or taxa)
=pod

=item B<-taxon taxon>

Name of a taxon for which all organisms will be downloaded.

The option -taxon can be used iteratively on the command line to
specify multiple taxa.

 download-organism -v 1 -taxon Viridiplantae

=cut

} elsif ($arg eq "-taxon") {
	  push @taxa,  shift(@arguments);


=pod

=item B<-listall>

List all organisms currently available in remote RSAT servers/genome repositories.

=cut

} elsif ($arg eq "-listall") {
          $main::listall = 1;

=pod

=item B<-servers>

List current remote RSAT servers/genome repositories.

=cut

} elsif ($arg eq "-servers") { 
          $main::servers = 1;

=pod

=item B<-no_blast> |  B<-blast>

Options to specify whether the cross-species blast files should be
downloaded or not together with the genome. 

Default: -no_blast

These files are used by several comparative genomics programs
(I<get-orthologs>, I<footprint-discovery>, I<footprint-scan>...). With
the increase of th number of sequenced genomes, the blast-hit folder
occupies a large disk space, and should be downloaded only

=cut

} elsif ($arg eq "-no_blast") {
    $main::no_blast = 1;

} elsif ($arg eq "-blast") {
    $main::no_blast = 0;

=pod

=item	B<-dir output directory>

Directory for exporting the genome(s) of the selected organism(s).  

By default, genomes are copied in the standard RSAT genome folder
($RSAT/public_html/data/genomes/). This requires write permissions on
this folder. If you don't have those writing permissions, specifying
an alternative directory may be convenient for obtaining the data
files, but the downloaded genoems will not be supported on the local
machine unless the configuration file is adapted (and this anyways
requires the writing permission in the RSAT directory).

=cut
	} elsif ($arg eq "-dir") {
	    $main::local_genomes_dir = shift(@arguments);


=pod

=item	B<-o outputfile>

The output file contains timing and information about downloaded
organisms (installation directories, ...).

If no output file is specified, the standard output is used.  This
allows to use the command within a pipe.

=cut
    } elsif ($arg eq "-o") {
      $main::outfile{output} = shift(@arguments);

=pod

=item B<-url server/>

Specify an alternative server or genome repository,
try -servers to list them and see default.

=cut

    } elsif ($arg eq "-url") {
      $RSAT_SERVER = shift(@arguments);

=pod

=item B<-skip>

Skip the N first organisms of the list. This option is useful for
resuming an interrupted download.

=cut
	} elsif ($arg eq "-skip") {
	    $main::skip_org = shift(@arguments);

=pod

=item B<-last>

Stop after the N first organisms of the list. This option is useful
for testing the options with taxon-wise queries.

=cut
	} elsif ($arg eq "-last") {
	    $main::last_org = shift(@arguments);

=pod

=item B<-no_config>

Download (or upload) the files but do not run the configuration task
(install-organisms -task config).

=cut
	} elsif ($arg eq "-no_config") {
	    $main::no_config = 1;

=pod

=item B<-no_data>

Do not transfer (download or upload) the data. This option enables to
run the configuration after the data has been transferred
(install-organisms -task config).

=cut
	} elsif ($arg eq "-no_data") {
	    $main::no_data = 1;

=pod

=item B<-dry>

Dry run: print the commands but do not execute them (just for
testing).

=cut
	} elsif ($arg eq "-dry") {
	    $main::dry = 1;

	} else {
	    &FatalError(join("\t", "Invalid option", $arg));

	}
    }

=pod

=back

=cut

}

################################################################
## Verbose message
sub Verbose {
  print $main::out "; download-organism ";
  &PrintArguments($main::out);
  printf $main::out "; %-22s\t%s\n", "Program version", $program_version;
  if (%main::infile) {
    print $main::out "; Input files\n";
    while (my ($key,$value) = each %main::infile) {
      printf $main::out ";\t%-13s\t%s\n", $key, $value;
    }
  }
  if (%main::outfile) {
    print $main::out "; Output files\n";
    while (my ($key,$value) = each %main::outfile) {
      printf $main::out ";\t%-13s\t%s\n", $key, $value;
    }
  }
}


__END__