package EPrints::Plugin::Sword::Import::Broker_PMC;

# Based heavily on the NLMDTD Importer
# NLM DTD XML format EPrints importer
# Initially developed for use with EDINA
# Justin Bradley, EPrints Services, 17/06/2010

use strict;
use EPrints::Plugin::Sword::Import;
use EPrints::Plugin::Sword::Import::Broker_Common;
use EPrints::Plugin::Import::DefaultXML;
use LWP::UserAgent;
use IPC::Run qw(run timeout);

use Data::Dumper;

our @ISA = qw/EPrints::Plugin::Sword::Import::Broker_Common EPrints::Plugin::Import::DefaultXML EPrints::Plugin::Sword::Import/;

our %SUPPORTED_MIME_TYPES = ("application/zip" => 1,);

our %UNPACK_MIME_TYPES = ("application/zip" => "Sword::Unpack:::Zip",);

my $DEBUG = 0;

sub query_db
{
  return EPrints::Plugin::Sword::Import::Broker_Common::query_db(@_)
};

sub extract
{
  return EPrints::Plugin::Sword::Import::Broker_Common::extract(@_)
};

sub extract_with_attr
{
  return EPrints::Plugin::Sword::Import::Broker_Common::extract_with_attr(@_)
};

sub strip_xml
{
  return EPrints::Plugin::Sword::Import::Broker_Common::strip_xml(@_) 
};

sub new
{
    my ($class, %params) = @_;

    my $self = $class->SUPER::new(%params);

    $self->{name}    = "Sword Importer for PubMed Central (NLM DTD XML [with files]) into the OA-RJ Broker";
#    $self->{visible} = "all";
#    $self->{produce} = ['list/eprint'];
    $self->{provenance} = 'PubMed Central';
    return $self;
} ## end sub new


###        $opts{file} = $file;
###        $opts{mime_type} = $headers->{content_type};
###        $opts{dataset_id} = $target_collection;
###        $opts{owner_id} = $owner->get_id;
###        $opts{depositor_id} = $depositor->get_id if(defined $depositor);
###        $opts{no_op}   = is this a No-op?
###        $opts{verbose} = is this verbosed?
sub input_file
{
  my ($plugin, %opts) = @_;

  my $session = $plugin->{session};

  my $dir  = $opts{dir};
  my $mime = $opts{mime_type};
  my $file = $opts{file};

  my $NO_OP = $opts{no_op};

  my $LOGDIR = '/home/broker/eprints/var';
  my $logging = "|PMC_Sword_Import";

  # Let's find the XML file to import:
  unless (defined $SUPPORTED_MIME_TYPES{$mime})
  {
    $plugin->add_verbose("[ERROR] unknown MIME TYPE '$mime'.");
    $plugin->set_status_code(415);
    return undef;
  } ## end unless (defined $SUPPORTED_MIME_TYPES...)

  my $unpacker = $UNPACK_MIME_TYPES{$mime};

  my $tmp_dir;
  my $files;

  if (defined $unpacker)
  {
    $tmp_dir = EPrints::TempDir->new("swordXXX", UNLINK => 1);

    if (!defined $tmp_dir)
    {
      print STDERR
          "\n[SWORD-DEPOSIT] [INTERNAL-ERROR] Failed to create the temp directory!";
      $plugin->add_verbose("[ERROR] failed to create the temp directory.");
      $plugin->set_status_code(500);
      return undef;
    } ## end if (!defined $tmp_dir)

    $files = $plugin->unpack_files($unpacker, $file, $tmp_dir);

    unless (defined $files)
    {
      $plugin->add_verbose("[ERROR] failed to unpack the files");
      return undef;
    }

    my $candidates = $plugin->get_files_to_import($files, "text/xml");

    if (scalar(@$candidates) == 0)
    {
      $plugin->add_verbose("[ERROR] could not find the XML file");
      $plugin->set_status_code(400);
      return undef;
    } ## end if (scalar(@$candidates...))
    elsif (scalar(@$candidates) > 1)
    {

      # We have more than one possible xml file. Take 'mets.xml' if it exists,
      # otherwise just take the one with the 'highest' name, alphabetic-wise
      my @mets = grep /\bmets\.xml$/, @$candidates;
      
      if (scalar(@mets) == 1)
      {
        $$candidates[0] = $mets[0];
        $plugin->add_verbose(
          "[WARNING] there were more than one XML file in this archive, however I'm using the one called 'mets.xml'"
        );
      } elsif (scalar(@mets) > 1)
      {
        @$candidates = sort { length($a) cmp length($b) } @mets;
        $plugin->add_verbose(
          "[WARNING] there were more than one mets.XML file in this archive, however I'm using the one closest to the top of the 'tree'"
        );
      } else
      {
        @mets = sort @$candidates;
        $$candidates[0] = $mets[0];
        $plugin->add_verbose(
          "[WARNING] there were more than one XML file in this archive. I am using the one with the 'highest' name, alphabetic-wise (closer to 'z' than 'A')"
        );
      } ## end else [ if (scalar(@mets) == 1...)]
    } ## end elsif (scalar(@$candidates...))

    $file = $$candidates[0];
   
    # remove the file from the list of files
    @{$files} = grep !/^$file$/, @{$files}
  } ## end if (defined $unpacker)

  my $dataset_id   = $opts{dataset_id};
  my $owner_id     = $opts{owner_id};
  my $depositor_id = $opts{depositor_id};

  my $dataset = $session->get_archive()->get_dataset($dataset_id);

  if (!defined $dataset)
  {
    print STDERR
        "\n[SWORD-METS] [INTERNAL-ERROR] Failed to open the dataset '$dataset_id'.";
    $plugin->add_verbose(
                 "[INTERNAL ERROR] failed to open the dataset '$dataset_id'");
    $plugin->set_status_code(500);
    return;
  } ## end if (!defined $dataset)

  my $fh;
  if (!open($fh, $file))
  {
    $plugin->add_verbose(
                       "[ERROR] couldnt open the file: '$file' because '$!'");
    $plugin->set_status_code(500);
    return;
  } ## end if (!open($fh, $file))

  my $unpack_dir;
  my $fntmp;

  $fntmp = $file;

  if ($fntmp =~ /^(.*)\/(.*)$/)
  {
    $unpack_dir = $1;
    $fntmp      = $2;
  }

  # needs to read the xml from the file:
  my ($l, $xml, $error);
  while (my $d = <$fh>)
  {
    $l .= $d;
  }
  close $fh;

  # Stuff the original xml through the LTG "black box" to mark up the 
  # recognised organisations (wich we use later on)
  my @process = ("/home/oarj/ltgstuff/ltg-oarj/scripts/process");
  IPC::Run::run \@process, \$l, \$xml, \$error, timeout( 10 ) or die "PMC LTG parse failure code: $?";


  my $dom_doc;
  eval { $dom_doc = EPrints::XML::parse_xml_string($xml); };

  if ($@ || !defined $dom_doc)
  {
    $plugin->add_verbose("[ERROR] failed to parse the xml: '$@'");
    $plugin->set_status_code(400);
    return;
  } ## end if ($@ || !defined $dom_doc...)

  if (!defined $dom_doc)
  {
    $plugin->{status_code} = 400;
    $plugin->add_verbose("[ERROR] failed to parse the xml.");
    return;
  } ## end if (!defined $dom_doc)

  my $dom_top = $dom_doc->getDocumentElement;


  unless (defined $dom_top && $dom_top)
  {
    $plugin->set_status_code(400);
    $plugin->add_verbose(
                  "[ERROR] failed to find root element of the XML.");
    return;
  } ## end if (!defined $xml_data...)

  my $epdata = $plugin->xml_to_epdata($dom_top);
  return unless (defined $epdata);

  # Add some useful (SWORD) info
  if (defined $depositor_id)
  {
    $epdata->{userid}          = $owner_id;
    $epdata->{sword_depositor} = $depositor_id;
  } else
  {
    $epdata->{userid} = $owner_id;
  }

  $epdata->{eprint_status} = $dataset_id;

  if ($NO_OP)
  {

    # need to send 200 Successful (the deposit handler will generate the XML response)
    $plugin->add_verbose(
                      "[OK] Plugin - import successful (but in No-Op mode).");
    $plugin->set_status_code(200);
    return;
  } ## end if ($NO_OP)

  my $eprint = $dataset->create_object($plugin->{session}, $epdata);

  unless (defined $eprint)
  {
    $plugin->set_status_code(500);
    $plugin->add_verbose("[ERROR] failed to create the EPrint object.");
    return;
  } ## end unless (defined $eprint)

  $logging .= "|".$eprint->get_id;

  # Embargo information
  my $embargo = "";
  if (exists $epdata->{'openaccess'} && exists $epdata->{'date'} )
  {
    if (lc ($epdata->{'openaccess'}) eq 'true' )
    {
      my ($yy, $mm, $dd) = split /-/, $epdata->{'date'};
      $mm += 6;
      if ($mm > 12)
      {
        $mm -= 12;
        $yy++;
      }
      $embargo = sprintf("%04d-%02d-%02d", $yy, $mm, $dd);
    }
  }

  # Go through all the files remaining from the .zip and add them
  if (scalar @{$files})
  {
    my $doc_data = {};
    foreach my $file (@$files)
    {
      chomp $file;

      # we ignore "manifest.txt" files
      next if ($file =~ /manifest\.txt$/);

      $file =~ m/^(\w+)/;
      my $prot = $1 if $1;
      $file =~ m/\/([^\/]+)$/; # find everything after the last slash
      my $filename = $1 if $1;

      # if the file is not a web reference, add 'file://' and the unpack
      # path to the filename
      for ($prot)
      {
        /http/ && do { last; };    # $filename stays unchanged

        { # default option
          $file =~ "file://$unpack_dir/$filename";
          last;
        };
      } ## end for ($prot)

      $doc_data->{_parent} = $eprint unless defined $doc_data->{_parent};
      $doc_data->{eprintid} = $eprint->get_id
                unless defined $doc_data->{eprintid};
      $doc_data->{main} = $filename unless $doc_data->{main};
      $doc_data->{format} = $session->get_repository->call(
                     'guess_doc_type', $session, $filename)
                unless exists $doc_data->{format};
      if ($embargo)
      {
        $doc_data->{date_embargo} = $embargo;
        $doc_data->{security} = 'staffonly';
      }

      my %file_data;
      $file_data{filename} = $filename;
      $file_data{url}      = $file;

      $doc_data->{files} = [] unless exists $doc_data->{files};
      push @{$doc_data->{files}}, \%file_data;
          
      # Now create the document (PMC has each file as a separate document)
      my $doc_dataset = $session->get_repository->get_dataset("document");
      local $session->get_repository->{config}->{enable_web_imports}  = 1;
      local $session->get_repository->{config}->{enable_file_imports} = 1;
      my $document = EPrints::DataObj::Document->create_from_data($session,
                                                   $doc_data,
                                                   $doc_dataset);
      unless( $document )
      {
        $plugin->add_verbose("[WARNING] Failed to create Document object(s).");
      }
    } ## end foreach my $fptr (@nodes)
  }

  if ($plugin->keep_deposited_file())
  {
    if (
       $plugin->attach_deposited_file($eprint, $opts{file}, $opts{mime_type}))
    {
      $plugin->add_verbose("[OK] attached deposited file.");
    } else
    {
      $plugin->add_verbose("[WARNING] failed to attach the deposited file.");
    }
  } ## end if ($plugin->keep_deposited_file...)

  $plugin->add_verbose("[OK] EPrint object created.");

  # Keep a track of the logging
  my $log_string = scalar(CORE::localtime)."$logging";
  open (LOG, ">>$LOGDIR/transfer_log") 
        or warn("could not open log file $LOGDIR/transfer_log");
  print LOG "$log_string\n";
  close (LOG);
  # and let's put it in the error_log too since it's dead useful
  $plugin->{session}->log("logging|$log_string");

  return $eprint;

} ## end sub input_file

sub xml_to_epdata
{
    my ($plugin, $xml_root) = @_;

    my $xml = ($xml_root->getElementsByTagName('front'))[0];

    my $epdata = {};
    my %orgs_lookup = ();

    $epdata->{provenance} = $plugin->{provenance};

    # We trust the source and we know it's peer-reviewed & published
    $epdata->{refereed}    = "TRUE";
    $epdata->{ispublished} = "pub";

    my $article_type = "article";   # for now we always want article as the type
         #my $article_type = $xml->getAttribute("article-type");
         #$epdata->{type} = $plugin->type_mapping($article_type);
         #$epdata->{type} = 'article' if not defined $epdata->{type};

    my $journal_meta = $xml->getElementsByTagName("journal-meta")->item(0);
    if (defined $journal_meta)
    {

        $plugin->extract("journal-title", "publication", $journal_meta,
                         $epdata);
        $plugin->extract_with_attr(
                                   "journal-id",      "publisher",
                                   "journal-id-type", "publisher",
                                   $journal_meta,     $epdata
                                  );
        $plugin->extract_with_attr("issn", "issn", "pub-type", "epub",
                                   $journal_meta, $epdata)
          ;    # using ppub rather than epub
        $plugin->extract("isbn", "isbn", $journal_meta, $epdata);
    } ## end if (defined $journal_meta...)
    else
    {
        print STDERR "Failed to find <journal-meta>, bailing.\n";
        return;    # bail invalid xml
    }

    my $article_meta = $xml->getElementsByTagName("article-meta")->item(0);

    unless (defined $article_meta)

      # CG prefers the short error situation to come first, and then the main
      # body of code to disappear off the bottom of the page.
    {
        print STDERR "Failed to find <article-meta>, bailing.\n";
        return;    # bail, invalid xml
    } ## end unless (defined $article_meta...)
    else
    {
        $plugin->extract_with_attr(
                                   "article-id",  "id_number",
                                   "pub-id-type", "doi",
                                   $article_meta, $epdata
                                  );
        $plugin->extract("article-id", "id_number", $article_meta, $epdata)
          if !defined $epdata->{id_number};

        $plugin->extract("article-title", "title",    $article_meta, $epdata);
        $plugin->extract("volume",        "volume",   $article_meta, $epdata);
        $plugin->extract("issue",         "number",   $article_meta, $epdata);
        $plugin->extract("abstract",      "abstract", $article_meta, $epdata);

        $plugin->extract("copyright-holder", "_copyright_holder", $article_meta,
                         $epdata);
        push @{$epdata->{copyright_holders}}, $epdata->{_copyright_holder}
          if defined $epdata->{_copyright_holder};

        $plugin->extract("fpage", "_fpage", $article_meta, $epdata);
        $plugin->extract("lpage", "_lpage", $article_meta, $epdata);
        $epdata->{pagerange} = $epdata->{_fpage} . "-" . $epdata->{_lpage}
          if defined $epdata->{_fpage} && $epdata->{_lpage};

        # Open Access is true by default
        my $oa = $article_meta->getElementsByTagName('open-access')->item(0);
        my $oa2 = $plugin->strip_xml($oa, 'open-access') if defined $oa;
        $epdata->{openaccess} = lc ($oa2) eq 'no' ? 'FALSE' : 'TRUE';

        # Firstly, get all the contributing authors, then also get any
        # corresponding authors!

        # Contributing Authors
        foreach my $contrib ($article_meta->getElementsByTagName("contrib"))
        {
            next
              unless $contrib->getAttribute("contrib-type") eq
                  "author";    # go with author only for now
            my @authors = $contrib->getElementsByTagName("name");
            if (scalar(@authors) == 0)
            {
                print STDERR
                  "found and author section with no <name> blocks, skipping.\n"
                  if $DEBUG;
                next;
            } ## end if (scalar(@authors) ==...)
            my $author = $authors[0];

            my $name = {};

            my $lastname = $author->getElementsByTagName("surname")->item(0);
            $name->{family} = $plugin->xml_to_text($lastname)
              if defined $lastname;

            # No point in going any further unless we have a last name!
            next unless $lastname;

            my $forename =
              $author->getElementsByTagName("given-names")->item(0);
            $name->{given} = $plugin->xml_to_text($forename)
              if defined $forename;

            my @xref_values;

            foreach my $xref ($contrib->getElementsByTagName("xref"))
            {
                next unless $xref->getAttribute("ref-type") eq "aff";
                my $rid = $xref->getAttribute("rid");

                my %org = ();

                # NB: depending upon the scheme, you may want to just use the
                # $rid, rather than resolving it here we are resolving it,
                # but only using the first entry

                foreach my $aff ($article_meta->getElementsByTagName("aff"))
                {
                    if ($aff->getAttribute("id") eq $rid)
                    {
                            $org{'rid'} = $rid;
                            $org{'institution'} =
                              $plugin->strip_xml($aff, "aff");
                    } ## end if ($aff->getAttribute...)
                } ## end foreach my $aff ($article_meta...)
                push @xref_values, \%org;

            } ## end foreach my $xref ($contrib->getElementsByTagName...)

            # Note that EPrints stores multiple fields in referenced lists,
            # so you need to record an empty item if there is an undefined
            # value, otherwise your counts get out of step
            my ($inst);
            foreach my $org (@xref_values)
            {
                $inst    = $org->{'institution'} ? $org->{'institution'} : '';
                 last;
            } ## end foreach my $org (@xref_values...)

            push @{$epdata->{creators_institution}}, $inst;
            push @{$epdata->{creators_name}},        $name;

            $orgs_lookup{$orgid} = $orgname;

        } ## end foreach my $contrib ($article_meta...)
        
        # Corresponding Author
        foreach my $corresp ($article_meta->getElementsByTagName("corresp"))
        {
           my $corresp_email = ${$corresp->getElementsByTagName("email")}[0];
           if ($corresp_email)
               $epdata->{contact_email} = 
                                   $plugin->xml_to_text($corresp_email);
           }
        }

        foreach my $pub_date ($article_meta->getElementsByTagName("pub-date"))
        {
            next
              unless $pub_date->getAttribute("pub-type") eq "epub"
            ;    # go with epub for now, rather than nihms-submitted, ppub,
                 # pmc-release

            my $d     = $pub_date->getElementsByTagName("day")->item(0);
            my $day   = $plugin->xml_to_text($d) if defined $d;
            my $m     = $pub_date->getElementsByTagName("month")->item(0);
            my $month = $plugin->xml_to_text($m) if defined $m;
            my $y     = $pub_date->getElementsByTagName("year")->item(0);
            my $year  = $plugin->xml_to_text($y) if defined $y;

            $epdata->{"date"} =
              sprintf("%04d-%02d-%02d", $year,
                                        $month? $month : '01',
                                        $day ? $day : '01');
            $epdata->{"date_type"} = "published";

        } ## end foreach my $pub_date ($article_meta...)
    } ## end else


    foreach my $key (keys %$epdata)
    {
        if ($key =~ /^_/)
        {
            delete $epdata->{$key};
        }
        else
        {
            print STDERR "$key=$epdata->{$key}\n" if $DEBUG;
        }

    } ## end foreach my $key (keys %$epdata...)

    # Some final tidying up
    # If we have an ISSN, and no publisher, try to get it from ROMEO

    if ($epdata->{issn} and not $epdata->{publisher})
    {
        _get_publisher_from_romeo($plugin, $epdata);
    }

    return $epdata;
} ## end sub xml_to_epdata

sub _get_publisher_from_romeo
{
    my ($plugin, $epdata) = @_;

    my $issn  = $epdata->{issn};
    return unless $issn;

    my $query = "http://www.sherpa.ac.uk/romeo/api24.php?issn=$issn";

    my $ua = LWP::UserAgent->new();
    $ua->timeout(10);
    my $response = $ua->get($query);

    my $content;
    if ($response->is_success)
    {
        $content = $response->decoded_content;

        # convert the XML text to ax XML DOM
        my $dom_doc;
        eval { $dom_doc = EPrints::XML::parse_xml_string($content) };

        if ($@ || !defined $dom_doc)
        {
            warn "Failed to parse ROMEO response: '$@'.";
            return;
        }

        if (!defined $dom_doc)
        {
            warn "Failed to parse ROMEO response";
            return;
        }

        # element-to-find, epdata_field, xml, epdata hash_ref
        $plugin->extract("romeopub", "publisher", $dom_doc, $epdata);

    } ## end if ($response->is_success...)
    else
    {
        warn $response->status_line;
        return;
    }

} ## end sub _get_publisher_from_romeo



1;
