# =-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=
# Mobius Forensic Toolkit
# Copyright (C) 2008,2009,2010,2011,2012,2013,2014,2015,2016,2017,2018,2019,2020 Eduardo Aguiar
#
# This program is free software; you can redistribute it and/or modify it
# under the terms of the GNU General Public License as published by the
# Free Software Foundation; either version 2, or (at your option) any later
# version.
#
# This program is distributed in the hope that it will be useful, but
# WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General
# Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
# =-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=
import pymobius.app.chromium
import pymobius.app.gecko
import pymobius.app.internet_explorer
import mobius
import datetime
import traceback

ANT_ID = 'visited-urls'
ANT_NAME = 'Visited URLs'
ANT_VERSION = '1.0'

# =-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=
# @brief Generic visited URL class
# =-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=
class VisitedURL (object):

  # =-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=
  # @brief Initialize object
  # =-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=
  def __init__ (self):
    self.is_encrypted = False
    self.is_deleted = False
    self.last_access_time = None
    self.expiration_time = None

# =-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=
# @brief Ant: visited-urls
# @author Eduardo Aguiar
# =-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=
class Ant (object):

  # =-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=
  # @brief Initialize object
  # =-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=
  def __init__ (self, item):
    self.id = ANT_ID
    self.name = ANT_NAME
    self.version = ANT_VERSION
    self.__item = item

  # =-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=
  # @brief Run ant
  # =-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=
  def run (self):
    if not self.__item.datasource:
      return

    self.__entries = []

    # @begin-deprecated since: 1.21
    if self.__retrieve_old_visited_urls ():
      pass

    else:
      # @end-deprecated

      self.__retrieve_chromium ()
      self.__retrieve_gecko ()
      self.__retrieve_internet_explorer ()

    self.__save_data ()

  # =-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=
  # @brief Retrieve data from Chromium based browsers
  # =-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=
  def __retrieve_chromium (self):
    try:
      model = pymobius.app.chromium.model (self.__item)
    
      for profile in model.get_profiles ():
        self.__retrieve_chromium_profile (profile)
    except Exception, e:
      mobius.core.log ('WRN ant.visited_urls (__retrieve_chromium): ' + str (e) + ' ' + traceback.format_exc ())
      
  # =-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=
  # @brief Retrieve data from Chromium based browsers
  # =-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=
  def __retrieve_chromium_profile (self, profile):
    try:
      for entry in profile.get_history ():
        if not entry.url.startswith ('file://'):
           v = VisitedURL ()
           v.timestamp = entry.timestamp
           v.url = entry.url
           v.title = entry.title
           v.profile = profile
           
           v.metadata = mobius.pod.map ()
           v.metadata.set ('id', entry.id)
           
           self.__entries.append (v)
    except Exception, e:
      mobius.core.log ('WRN ant.visited_urls (__retrieve_chromium_profile): ' + str (e) + ' ' + traceback.format_exc ())

  # =-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=
  # @brief Retrieve data from Gecko based browsers
  # @see http://doxygen.db48x.net/mozilla/html/interfacensIDownloadManager.html
  # =-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=
  def __retrieve_gecko (self):
    model = pymobius.app.gecko.model (self.__item)
    
    for profile in model.get_profiles ():
      for entry in profile.get_history ():
        if not entry.url.startswith ('file://'):
           v = VisitedURL ()
           v.timestamp = entry.timestamp
           v.url = entry.url
           v.title = entry.title
           v.profile = profile
           
           v.metadata = mobius.pod.map ()
           v.metadata.set ('id', entry.id)
           v.metadata.set ('typed', entry.typed)
           v.metadata.set ('visit-type', entry.visit_type)
           
           self.__entries.append (v)

  # =-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=
  # @brief Retrieve data from Internet Explorer
  # @todo Move Unicode convertion to profile class
  # =-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=
  def __retrieve_internet_explorer (self):
    model = pymobius.app.internet_explorer.model (self.__item)
    
    for profile in model.get_profiles ():
      for entry in profile.get_history ():
        if not entry.url.startswith ('file://') and entry.timestamp_utc:
           v = VisitedURL ()
           v.timestamp = entry.timestamp
           v.url = entry.url
           v.title = entry.title or ''
           v.profile = profile
           
           v.metadata = mobius.pod.map ()
           v.metadata.set ('file-path', entry.index_dat_path)
           v.metadata.set ('file-type', entry.file_type)
           v.metadata.set ('file-creation-time', entry.index_dat_creation_time)
           v.metadata.set ('file-last-modification-time', entry.index_dat_last_modification_time)
           v.metadata.set ('record-offset', '0x%08x' % entry.offset)
           v.metadata.set ('record-type', entry.type)
           v.metadata.set ('record-size', entry.size)
           v.metadata.set ('record-primary-time', entry.primary_time)
           v.metadata.set ('record-secondary-time', entry.secondary_time)
           
           local_time = entry.tags.get (0x18)
           if local_time:
             v.metadata.set ('local-time-tag-0x18', local_time)

           if entry.type == 'URL':
             v.metadata.set ('expiration-time', entry.expiration_time)
             v.metadata.set ('last-sync-time', entry.last_sync_time)
             v.metadata.set ('hits', entry.hits)
             v.metadata.set ('cached-file-path', entry.filename)
           
           self.__entries.append (v)

  # =-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=
  # @brief Retrieve data from 'browser.json' file
  # @deprecated since: 1.21
  # =-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=
  def __retrieve_old_visited_urls (self):
          
    # check if browser.json file exists
    path = self.__item.get_data_path ('browser.json')
    f = mobius.io.new_file_by_path (path)

    if not f.exists ():
      return False

    # unserialize file
    reader = f.new_reader ()
    data = pymobius.json_serializer.unserialize (reader.read ())

    # check if data is valid
    if data.mtime < self.__item.datasource.mtime or not hasattr (data, 'history'):
      return False
        
    # retrieve data
    self.__retrieve_old_data (data.history)

    # remove history from data
    if data.profiles == None and data.form_history == None:
      os.remove (path)
      return True

    del data.history
    data.mtime = datetime.datetime.utcnow ()
    
    # create file
    fp = open (path, 'w')

    try:
      fp.write (pymobius.json_serializer.serialize (data))
      fp.close ()

    except Exception, e:
      mobius.core.log ('WRN ant.visited_urls (__save_data): '  + str (e) + ' ' + traceback.format_exc ())
      fp.close ()
      os.remove (path)
      return False
    
    return True

  # =-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=
  # @brief Retrieve data from 'browser.json' history data
  # @deprecated since: 1.21
  # =-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=
  def __retrieve_old_data (self, history):

    def iso_to_datetime (s):
      return datetime.datetime.strptime (s, '%Y-%m-%d %H:%M:%S')

    class profile (object): pass
    profiles = {}

    for h in history:
      metadata = dict ((k.lower (), v) for k, v in h.metadata)

      v = VisitedURL ()
      v.timestamp = iso_to_datetime (h.timestamp)
      v.url = h.url
      v.title = metadata.get ('page title')
      v.metadata = mobius.pod.map ()

      # last access time
      last_access_time = metadata.get ('last access date/time')
      if last_access_time:
        v.last_access_time = iso_to_datetime (last_access_time)

      # expiration time
      expiration_time = metadata.get ('expiration date/time')
      if expiration_time:
        v.metadata.set ('expiration-time', iso_to_datetime (expiration_time))

      # evidence_path
      if h.app in ('chrome', 'opera'):
        evidence_path = h.profile_path + u'History'
        
      elif h.app in ('firefox', 'geckofx'):
        evidence_path = h.profile_path + u'\\places.sqlite'

      elif h.app == 'ie':
        evidence_path = metadata.get ('index.dat file path')

      else:
        evidence_path = h.profile_path

      v.metadata.set ('evidence-path', evidence_path)

      for name, value in h.metadata:
        v.metadata.set (name, value)

      # profile
      p = profiles.get ((h.app, h.profile_path))

      if not p:
        p = profile ()
        p.name = h.profile_id
        p.path = h.profile_path
        p.creation_time = None
        p.app_id = h.app
        p.app_name = h.app_name
        p.username = h.username
        p.creation_time = None
        profiles[(h.app, h.profile_path)] = p

      v.profile = p
      self.__entries.append (v)

  # =-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=
  # @brief Save data into model
  # =-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=
  def __save_data (self):
    case = self.__item.case
    transaction = case.new_transaction ()

    # remove old data
    self.__item.remove_visited_urls ()

    # save entries
    profiles = {}

    for e in self.__entries:
      url = self.__item.new_visited_url (e.timestamp, e.url)
      url.title = e.title or ''
      url.metadata = e.metadata
      
      # create profile, if necessary
      p = profiles.get (id (e.profile))
      if not p:
        app = case.new_application (e.profile.app_id, e.profile.app_name)

        p = self.__item.new_profile (e.profile.app_id, e.profile.path)
        p.id = e.profile.name
        p.username = e.profile.username
        
        if e.profile.creation_time:
          p.creation_time = e.profile.creation_time
        profiles[id (e.profile)] = p

      url.profile = p

    # set ant run
    self.__item.set_ant (ANT_ID, ANT_NAME, ANT_VERSION)
    transaction.commit ()
