Logo Search packages:      
Sourcecode: kat version File versions  Download package

katindexer.cpp

/***************************************************************************
 *   Copyright (C) 2005 by Roberto Cappuccio and the Kat team              *
 *   Roberto Cappuccio : roberto.cappuccio@gmail.com                       *
 *   Praveen Kandikuppa : praveen9@gmail.com                               *
 *                                                                         *
 *   This program is free software; you can redistribute it and/or modify  *
 *   it under the terms of the GNU General Public License as published by  *
 *   the Free Software Foundation; either version 2 of the License, or     *
 *   (at your option) any later version.                                   *
 *                                                                         *
 *   This program is distributed in the hope that it will be useful,       *
 *   but WITHOUT ANY WARRANTY; without even the implied warranty of        *
 *   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the         *
 *   GNU General Public License for more details.                          *
 *                                                                         *
 *   You should have received a copy of the GNU General Public License     *
 *   along with this program; if not, write to the                         *
 *   Free Software Foundation, Inc.,                                       *
 *   51 Franklin Steet, Fifth Floor, Boston, MA 02110-1301, USA.           *
 ***************************************************************************/

#include <math.h>
#include <qapplication.h>
#include <qregexp.h>
#include <kdebug.h>

#include <katengine.h>
#include <katcatalog.h>
#include <katscanfolder.h>
#include <katinfoextractor.h>

#include "katdaemonevents.h"
#include "katindexer.h"
#include "katinfoextractor.h"
#include "kattemptable.h"
#include "katxattr.h"
#include "config.h"
bool ddebug = true;

#define CHECKEVENT(e) m_eventMutex.lock();checkWatcherEvent(e);m_eventMutex.unlock();

/* Points to note here
   1. All functions which call database must be done only after obtaining exclusive
      lock from scheduler.
   2. All codepaths which take time must be allowed to be paused/stopped.
*/

class WatcherEvent
{
public:
    WatcherEvent() {};
    WatcherEvent( QString path, KatIndexer::Action action ) : m_path(path), m_action(action), m_hold(1) {};

    QString m_path;
    KatIndexer::Action m_action;
    QTime m_time;
    int m_hold;
};

KatIndexer::KatIndexer( QObject* parent, KatEngine* e, KatCatalog* cat, KatTempTable* table, KatScheduler* scheduler )
    : QObject( parent, "KatIndexer" )
{
    m_tempTable = table;
    m_ke = e;
    m_catalog = cat;
    m_scheduler = scheduler;
    m_catalogId = m_catalog->catalogId();

    if( getenv ( "KAT_DAEMON_NODEBUG" ) != NULL )
        ddebug = false;

    m_running = false;
    m_pause = false;
    m_committing = false;

    m_schedWait = 0;

    m_watcher = 0;
    m_kie = 0;
    m_ksf = 0;
}

KatIndexer::~KatIndexer()
{
    // Wake up all blocking events
    m_running = false;
    m_schedWait = 0;

    if ( m_kie )
    {
        m_kie->slotAbortExtraction();
        m_kie->slotAbortSaveInfo();
    }

    m_schedCond.wakeAll();
    m_eventCond.wakeAll();
    m_sleepCond.wakeAll();
    m_pauseCond.wakeAll();
    m_infoCond.wakeAll();

    m_scheduler->releaseLock( m_catalogId, 0 );

    // Wait for this thread to finish
    if ( !wait( 1000 ) )
    {
        kdDebug() << m_catalogId << " " << "KILLING THIS THREAD" << endl;
        terminate();
    }

    cleanUp();

    // Delete table
    delete m_tempTable;
    m_tempTable = 0;
}

void KatIndexer::setIgnore( const QStringList& ignoreDirs, const QStringList& ignoreFiles )
{
    m_ignoreDirs = ignoreDirs;
    m_ignoreFiles = ignoreFiles;
}

void KatIndexer::customEvent( QCustomEvent* e )
{
    if ( e->type() == 9000 )
    {   //pause event
        if ( !m_pause && m_running )
        {
            m_pause = true;

            if ( m_kie )
            {
                m_kie->slotAbortExtraction();
                m_kie->slotAbortSaveInfo();
            }

            m_eventCond.wakeAll(); // wake it from waiting for events
            m_sleepCond.wakeAll();
            m_schedCond.wakeAll();
            m_infoCond.wakeAll();
        }
    }
    else if ( e->type() == 9001 )
    { //stop event

        m_running = false;

        if ( m_kie )
        {
            m_kie->slotAbortExtraction();
            m_kie->slotAbortSaveInfo();
        }

        m_eventCond.wakeAll();
        m_sleepCond.wakeAll();
        m_pauseCond.wakeAll();
        m_schedWait = 0;
        m_schedCond.wakeAll();
        m_infoCond.wakeAll();
    }
    else if ( e->type() == 9002 )
    { //resume event
        m_pause = false;
        m_schedCond.wakeAll();
        m_pauseCond.wakeAll();
    }
    else if ( e->type() == 9021 )
    {
        if ( !m_running || ( m_pause && !m_committing ) )
        {
            // We were not scheduled at all
            m_scheduler->releaseLock( m_catalogId, 0 );
            return;
        }
        ScheduleEvent* se = ( ScheduleEvent* )e;
        m_schedWait = se->waitTime();
        m_schedCond.wakeAll();
    }
}

/*******************************************************************************
 *************** SCAN RELATED ROUTINES *****************************************
 ******************************************************************************/

void KatIndexer::processDirOnMove ( QDir dir )
{
    if ( !m_running )
        return;

    if ( m_catalog->autoUpdate() == 2 )
    {
        KatWatcher::EventType watchMask = (KatWatcher::EventType) ( KatWatcher::Modify | KatWatcher::Attrib | KatWatcher::Move |
                KatWatcher::Delete | KatWatcher::Create | KatWatcher::DeleteSelf | KatWatcher::Unmount );

        if ( m_watcher && ( m_watcher->watch( dir.absPath(), watchMask ) >= 0 ) )
            kdDebug( ddebug ) << "Added inotify watch on directory " << dir.absPath().latin1() << endl;
        else
            kdDebug( ddebug ) << "Could not watch the directory " << dir.absPath().latin1() << endl;
    }

    // Watch successful -- Prepend list of sub directories to m_crawlQueue - this is important */
    // FIXME -- What about symlinks??
    // FIXME - what about .files
    QStringList subdirs = dir.entryList( QDir::Dirs | QDir::Readable | QDir::NoSymLinks | QDir::Hidden );
    QStringList ndirs;

    // remove "." and ".." entries
    subdirs.remove( "." );
    subdirs.remove( ".." );

    for ( QStringList::Iterator it = subdirs.begin(); it != subdirs.end(); ++it )
        ndirs << (*it).prepend( "/" ).prepend( dir.absPath() );

    // prune the list to remove m_ignoreDirs
    interesting( ndirs, true );

    // Add the subdir list to crawlQueue
    m_crawlQueue = ndirs + m_crawlQueue;

    // Now get the file list
    // FIXME -- What about symlinks??
    // FIXME - what about .files
    WatcherEvent event;

    // First queue the present directory
    event.m_path = dir.absPath();
    event.m_action = (KatIndexer::Action) ( KatIndexer::Create|KatIndexer::Update );
    event.m_hold = 1;
    CHECKEVENT ( event );

    QStringList files = dir.entryList( QDir::Files | QDir::NoSymLinks | QDir::Hidden );

    // prune the list to remove m_ignoreFiles
    interesting( files, false );

    // Now do the same for all its files
    for ( QStringList::Iterator it = files.begin(); it != files.end(); ++it )
    {
        event.m_path = ( *it ).prepend( "/" ).prepend( dir.absPath() );
        event.m_action = (KatIndexer::Action) ( KatIndexer::Create|KatIndexer::Update );
        event.m_hold = 1;
        CHECKEVENT ( event );
    }
}

int KatIndexer::processDir ( QDir dir )
{
    if ( !m_running )
        return 0;

    if ( m_catalog->autoUpdate() == 2 )
    {
        KatWatcher::EventType watchMask = (KatWatcher::EventType) ( KatWatcher::Modify | KatWatcher::Attrib | KatWatcher::Move |
                KatWatcher::Delete | KatWatcher::Create | KatWatcher::DeleteSelf | KatWatcher::Unmount );

        if ( m_watcher && ( m_watcher->watch( dir.absPath(), watchMask ) >= 0 ) )
            kdDebug( ddebug ) << "Added inotify watch on directory " << dir.absPath().latin1() << endl;
        else
            kdDebug( ddebug ) << "Could not watch the directory " << dir.absPath().latin1() << endl;
    }

    // Watch successful -- Prepend list of sub directories to m_crawlQueue - this is important */
    // FIXME -- What about symlinks??
    // FIXME -- what about .files
    QStringList subdirs = dir.entryList( QDir::Dirs | QDir::Readable | QDir::NoSymLinks | QDir::Hidden );
    QStringList ndirs;

    // remove "." and ".." entries
    subdirs.remove( "." );
    subdirs.remove( ".." );

    for ( QStringList::Iterator it = subdirs.begin(); it != subdirs.end(); ++it )
        ndirs << (*it).prepend( "/" ).prepend( dir.absPath() );

    // prune the list to remove m_ignoreDirs
    interesting( ndirs, true );

    // Add the subdir list to crawlQueue
    m_crawlQueue = ndirs + m_crawlQueue;

    // Now get the file list
    // FIXME -- What about symlinks??
    // FIXME -- what about .files
    QMap<QString, int> tempRecords;
    const QFileInfoList* filesList = dir.entryInfoList( QDir::Files | QDir::NoSymLinks | QDir::Hidden );

    QFileInfo *fileInfo = new QFileInfo( dir.absPath() );
    KatIndexer::Action action = findAction( fileInfo );
    tempRecords[dir.absPath()] = action;
    delete fileInfo;
    fileInfo = 0;

    // Then walk through all its children
    QFileInfoListIterator it( *filesList );
    while ( (fileInfo = it.current()) != 0 )
    {
        action = findAction( fileInfo );
        if ( ( action != KatIndexer::None ) && interesting( fileInfo->absFilePath(), false ) )
            tempRecords[ fileInfo->absFilePath() ] = action;
        ++it;
    }

    // Now add these to the table
    m_tempTable->addRecords( tempRecords );

    return ( filesList->count() + 1 );

}

void KatIndexer::crawlOnMove ( QString path )
{
    m_crawlQueue = path;

    while ( !m_crawlQueue.empty() && m_running ) {

        /* Remove the first item from the queue */
        QString dir = m_crawlQueue[0];
        m_crawlQueue.pop_front();

        if ( dir.isEmpty() )
            continue;

        QFileInfo baseFile( dir );

        if ( !baseFile.exists() || !baseFile.isDir() ) {
            kdDebug( ddebug ) << "The directory to be watched does not exist or is not a directory " << dir << endl;
            continue;
        }

        processDirOnMove ( QDir( baseFile.absFilePath() ) );
    }
}

void KatIndexer::crawl ( QString path )
{
    m_crawlQueue = path;

    QTime scanTime;

    bool scheduled = false;
    int nFiles = 0;

    if ( parent() )
        QApplication::postEvent( parent(), new StatusEvent( m_catalogId, KatIndexer::Scan ) );


    while ( !m_crawlQueue.empty() && m_running ) {

        // Pause here if the status is pause
        if ( m_pause ) {

            // release schedule
            if ( scheduled )
            {
                m_scheduler->releaseLock( m_catalogId, scanTime.elapsed() );
                scheduled = false;
            }

            if ( parent() )
                QApplication::postEvent( parent(), new StatusEvent( m_catalogId, KatIndexer::Pause ) );

            // Store scan state
            // Store m_processFiles state;
            slotCommitTimeout();

            if ( m_running && m_pause )
                m_pauseCond.wait();

            if ( !m_running )
                break;

            // Stop the commitTimer
            m_commitTimer.stop();

            if ( parent() )
                QApplication::postEvent( parent(), new StatusEvent( m_catalogId, KatIndexer::Scan ) );
        }

        if ( !scheduled )
        {
            if ( acquireLock ( KatScheduler::Normal ) )
            {
                scanTime.start();
                scheduled = true;
            }
            else
            {
                if ( !m_running )
                    break;
                else
                    continue;
            }
        }

        /* Remove the first item from the queue */
        QString dir = m_crawlQueue[0];
        m_crawlQueue.pop_front();

        if ( dir.isEmpty() )
            continue;

        QFileInfo baseFile( dir );

        if ( !baseFile.exists() || !baseFile.isDir() )
        {
            kdDebug( ddebug ) << "The directory to be watched does not exist or is not a directory " << dir << endl;
            continue;
        }

        // Send CurrentFileEvent to indicate the directory being processed
        if ( parent() )
            QApplication::postEvent( parent(), new CurrentFileEvent( m_catalogId, dir, 1 ) );

        nFiles += processDir ( QDir( baseFile.absFilePath() ) );

        // Send ProgressEvent to send the number of files/dirs scanned
        if ( parent() )
            QApplication::postEvent( parent(), new ProgressEvent( m_catalogId, nFiles, 0 ) );
    }

    if ( scheduled )
    {
        m_scheduler->releaseLock( m_catalogId, scanTime.elapsed() );
        scheduled = false;
    }

    if ( !m_running )
        return;

    // Send One last progressEvent to send the number of files/dirs scanned
    if ( parent() )
        QApplication::postEvent( parent(), new ProgressEvent( m_catalogId, m_tempTable->numRecords(), 0 ) );
}

void KatIndexer::scanRoutine()
{
    if ( !m_running )
        return;

    kdDebug() << m_catalogId << " " << k_funcinfo << endl;

    // Initialize the crawl queue
    m_crawlQueue = m_catalog->path();

    // FIXME -- Get the ignore dirs list from catalog

    crawl( m_catalog->path() );

    kdDebug() << m_catalogId << " " << k_funcinfo << " done" <<  endl;
}

bool KatIndexer::acquireLock( KatScheduler::Priority pri )
{
    kdDebug( ddebug ) << m_catalogId << " requesting a schedule" << endl;

    // Request a schedule
    m_schedWait = m_scheduler->requestLock( this, m_catalogId, pri );

    // Wait here for us to be scheduled
    if ( m_schedWait == -1 )
    {
        if ( parent() )
            QApplication::postEvent( parent(), new SubStatusEvent( m_catalogId, KatIndexer::RequestSchedule ) );
        m_schedCond.wait();
    }

    if ( !m_running || m_pause )
    {
        // Release lock
        m_scheduler->releaseLock( m_catalogId, 0 );
        if ( parent() )
            QApplication::postEvent( parent(), new SubStatusEvent( m_catalogId, KatIndexer::Reset ) );
        return false;
    }

    kdDebug( ddebug ) << m_catalogId << " has  been scheduled for " << m_schedWait << endl;

    // We have been scheduled
    if ( m_schedWait )
    {
        if ( parent() )
            QApplication::postEvent( parent(), new SubStatusEvent( m_catalogId, KatIndexer::ScheduleTimeout, QVariant( m_schedWait ) ) );

        // FIXME -- Check that on Stop/Pause this is woken up
        m_schedCond.wait( m_schedWait );
    }

    if ( parent() )
        QApplication::postEvent( parent(), new SubStatusEvent( m_catalogId, KatIndexer::Reset ) );

    if ( !m_running || m_pause )
    {
        // Release lock
        m_scheduler->releaseLock( m_catalogId, 0 );
        return false;
    }
    else
        return true;
}

void KatIndexer::indexRoutine()
{
    if ( !m_running )
        return;

    QTime itime;    // Index time
    long totITime = 0;
    int filesDone = 0;
    int totIndexFiles = m_tempTable->numRecords();

    ProcessMap iprocessFiles;

    if ( parent() )
        QApplication::postEvent( parent(), new StatusEvent( m_catalogId, Index ) );

    m_tempTable->startReadSession();

    while ( m_running ) {
        if ( m_pause ) {
            if ( parent() )
                QApplication::postEvent( parent(), new StatusEvent( m_catalogId, Pause ) );

            // Save state
            slotCommitTimeout();

            // NOTE - Check m_pause here beacuse -- we have been sent a pause request
            // try to save the current state but before the state has been saved
            // a resume request is sent, in that case this will block below
            // Checking m_pause will prevent that
            if ( m_running && m_pause )
                m_pauseCond.wait();

            if ( m_running )
            {
                // Stop the commitTimer
                m_commitTimer.stop();

                if ( parent() )
                    QApplication::postEvent( parent(), new StatusEvent( m_catalogId, Index ) );
            }
        }

        itime.start();

        iprocessFiles.clear();
        // Now process the files we have pulled from the table
        if ( !acquireLock ( KatScheduler::Immediate ) )
            continue;

        iprocessFiles = m_tempTable->readRecords ( 20 );

        m_scheduler->releaseLock( m_catalogId, 0 );

        if ( !iprocessFiles.empty() )
        {
            // Now take care of events accumulated while we were indexing
            if ( !m_waitingEvents.empty() )
                processWaiting();

            // Add iprocessFiles to m_processFiles and send it to processFiles()
            m_processMutex.lock();
            for( ProcessMap::Iterator it = iprocessFiles.begin(); it != iprocessFiles.end(); ++it )
                m_processFiles[it.key()] = it.data();
            m_processMutex.unlock();

            // CurrentFileEvent will be taken care of by processFiles()
            if ( !processFiles() )
            {
                kdDebug() << m_catalogId << " " << " we have been paused while indexing .. " << endl;
                continue;
            }

            filesDone += 20;
        }

        totITime += itime.elapsed();

        // Update progress
        if ( ( totIndexFiles < filesDone ) || iprocessFiles.empty() )
        {
            if ( parent() )
                QApplication::postEvent( parent(), new ProgressEvent( m_catalogId, totIndexFiles, totITime ) );
            break;
        }
        else
        {
            if ( parent() )
                QApplication::postEvent( parent(), new ProgressEvent( m_catalogId, filesDone, totITime ) );
        }

        kdDebug( ddebug ) << "Starting next index cycle " << endl;
    }

    kdDebug() << m_catalogId << " " << " Indexing all the files/folders in catalog " << m_catalogId << " took " << totITime << " seconds" << endl;

    if ( !m_running )
        return;

    // clear m_tempTable
    while ( !acquireLock ( KatScheduler::Immediate ) )
    {
        if ( !m_running )
            break;

        if ( m_pause )
        {
            if ( parent() )
                QApplication::postEvent( parent(), new StatusEvent( m_catalogId, Pause ) );

            m_pauseCond.wait();
        }
    }

    if ( m_running )
    {
        m_tempTable->clearTable();
        m_scheduler->releaseLock( m_catalogId, 0 );
    }
}

void KatIndexer::run()
{
    int hold;

    if ( !m_scheduler )
    {
        kdDebug() << m_catalogId << " " << " A scheduler is not assigned to the indexer - will not index " << endl;
        return;
    }

    if ( m_catalog->autoUpdate() == 2 )
    {
        m_watcher = new Inotify( this, "kat_inotify", QStringList() );

        if ( m_watcher->enabled() )
        {
            // Set the Watcher slots
            connect( m_watcher, SIGNAL( onEvent( KatWatcher::EventType, QString, QString ) ), this, SLOT( slotOnEvent( KatWatcher::EventType, QString, QString ) ) );

            // start watcher
            m_watcher->startWatcher();
        }
        else
        {
            delete m_watcher;
            m_watcher = 0;
        }
    }

    m_kie = m_ke->extractInfo( m_catalog );
    connect( m_kie, SIGNAL( completed() ), this, SLOT( slotInfoExtractorCompleted() ) );

    m_ksf = m_ke->scanFolder( m_catalog );

    connect( &m_commitTimer, SIGNAL( timeout() ), this, SLOT( slotCommitTimeout() ) );

    m_running = true;

    // Request a schedule to prune catalog
    m_schedWait = m_scheduler->requestLock( this, m_catalogId, KatScheduler::Immediate );

    // Wait here for us to be scheduled
    if ( m_schedWait == -1 )
    {
        if ( parent() )
            QApplication::postEvent( parent(), new SubStatusEvent( m_catalogId, RequestSchedule ) );

        m_schedCond.wait();
    }

    // We have been scheduled
    if ( m_schedWait )
    {
        if ( parent() )
            QApplication::postEvent( parent(), new SubStatusEvent( m_catalogId, ScheduleTimeout, QVariant( m_schedWait ) ) );

        m_schedCond.wait( m_schedWait );
    }

    if ( parent() )
        QApplication::postEvent( parent(), new SubStatusEvent( m_catalogId, Reset ) );

    // Pausing here is same as stopping
    if ( !m_running || m_pause )
    {
        m_scheduler->releaseLock( m_catalogId, 0 );
        cleanUp();

        if ( parent() )
            QApplication::postEvent( parent(), new StatusEvent( m_catalogId, Stop ) );

        // return here
        return;
    }

    // Delete the files in database which jave been deleted from FileSystem
    if ( parent() )
        QApplication::postEvent( parent(), new StatusEvent( m_catalogId, Prune ) );

    m_ke->pruneCatalog (m_catalog);
    m_tempTable->clearTable();

    m_scheduler->releaseLock( m_catalogId, 0 );

    // Start Scanning
    scanRoutine();

    indexRoutine();

    if ( m_catalog->autoUpdate() == 2 )
    {
        while ( m_running ) {

            if ( m_pause ) {
                // Post the event that we are paused
                if ( parent() )
                    QApplication::postEvent( parent(), new StatusEvent( m_catalogId, Pause ) );

                //commit remaining events in dispatch and waiting queues to table
                slotCommitTimeout();

                if ( m_running && m_pause )
                    m_pauseCond.wait();

                if ( m_running ) // Take care of all the events which have crawled up when we were paused
                    indexRoutine();
                else
                    break;
            }

            if ( m_processFiles.empty() && m_waitingEvents.empty() ) {
                kdDebug( ddebug ) << "Waiting for inotify events in Indexer thread " << endl;

                if ( parent() )
                    QApplication::postEvent( parent(), new StatusEvent( m_catalogId, Wait ) );

                m_eventCond.wait();

                if ( !m_running ) // Take care of all the events which have crawled up when we were paused
                    break;

                if ( m_pause )
                    continue;

                kdDebug( ddebug ) << "Inotify events received, processing " << endl;
            }

            // We are here means we have some events to process
            if ( m_running && !m_pause ) {
                // Stop the commitTimer here, we don't need it anymore
                m_commitTimer.stop();

                if ( parent() )
                    QApplication::postEvent( parent(), new StatusEvent( m_catalogId, Process ) );
            }

            hold = 1;

            while ( m_running && !m_pause ) {

                // Go through the m_waitingEvents and dispatch expired events
                if ( !m_waitingEvents.empty() ) {

                    kdDebug( ddebug ) << "Sleeping for " << hold * 600 << " before beginning ... " << endl;

                    if ( hold < 200 )
                        m_sleepCond.wait( hold * 600 );
                    else
                        m_sleepCond.wait( 120000 );

                    hold = processWaiting();
                }

                if ( !m_processFiles.empty() )
                    processFiles();
                if ( m_processFiles.empty() && m_waitingEvents.empty() )
                    break;
            }
        }
    }

    m_scheduler->releaseLock( m_catalogId, 0 );

    // FIXME
    cleanUp();

    if ( parent() )
        QApplication::postEvent( parent(), new StatusEvent( m_catalogId, Stop ) );
}

void KatIndexer::cleanUp()
{
    if ( m_watcher && ( m_catalog->autoUpdate() == 2 ) )
    {
        m_watcher->stopWatcher();

        delete m_watcher;
        m_watcher = 0;
    }

    if ( m_kie )
    {
        delete m_kie;
        m_kie = 0;
    }

    if ( m_ksf )
    {
        delete m_ksf;
        m_ksf = 0;
    }

    disconnect( &m_commitTimer, 0, this, 0 );

    // FIXME - Reset all variables
    m_crawlQueue.clear();

    m_waitingEvents.clear();

    m_processFiles.clear();
    m_moveFiles.clear();

    m_schedWait = 0;

    m_commitTimer.stop();
}

int KatIndexer::processWaiting()
{
    QTime curTime = QTime::currentTime();
    int timeto;
    int hold;

    m_eventMutex.lock();
    m_processMutex.lock();
    hold = m_waitingEvents.begin().data().m_hold;
    for ( EventMap::Iterator it = m_waitingEvents.begin(); it != m_waitingEvents.end(); ++it ) {
        timeto = curTime.secsTo( it.data().m_time );
        if ( ( timeto < 0 ) || ( timeto > 3600 ) ) {
            m_processFiles[ it.key() ] = it.data().m_action;
            kdDebug( ddebug ) << "Removing event from queue " << it.key().latin1() << " <-> " << it.data().m_action << endl;
            m_waitingEvents.remove( it.key() );
        } else {
            if ( it.data().m_hold < hold )
                hold = it.data().m_hold;
            kdDebug( ddebug ) << "Holding event in queue - " << it.key().latin1() << " <-> " << it.data().m_action << endl;
        }
    }
    m_processMutex.unlock();
    m_eventMutex.unlock();

    return hold;
}

KatIndexer::Action KatIndexer::findAction ( const QFileInfo *fi )
{
    time_t lastupdate = 0;

    // If this is a directory, fetch from the catalog, because directory modification/creation
    //  times seem to be a little screwed
    // if ( m_catalog->useExtendedAttr() && !fi->isDir() )
#ifdef HAVE_ATTR_H
    if ( m_catalog->useExtendedAttr() )
        lastupdate = KatExtendedAttr::getIntExtendedAttribute( fi->absFilePath(), "lastupdatedate" );
#endif
    if ( !lastupdate )
        lastupdate =  m_ke->readFileInformation( fi->absFilePath().latin1(), KatInformation::FileInfo ).lastUpdateDate;

    //kdDebug( ddebug ) << lastupdate << " " << fi->created().toTime_t() << " " <<  fi->lastModified().toTime_t() << " " <<  fi->lastRead().toTime_t() << endl;
    if ( lastupdate )
    {
        if ( ( lastupdate < ( time_t ) fi->created().toTime_t() ) && !fi->isDir() )  // Means the file was moved since last update
            return ( KatIndexer::Action )( KatIndexer::Create|KatIndexer::Update );
        else if ( lastupdate < ( time_t ) fi->lastModified().toTime_t() )
            return KatIndexer::Update;
        else if ( lastupdate < ( time_t ) fi->lastRead().toTime_t() )
            return KatIndexer::Attrib;
        else
            return KatIndexer::None;
    }
    else
        return ( KatIndexer::Action )( KatIndexer::Create|KatIndexer::Update );
}

void KatIndexer::slotOnEvent ( KatWatcher::EventType eventtype, QString path, QString oldpath )
{
    if ( !m_running )
        return;

    QFileInfo fi( path );

    // Check the event against a blacklist
    if ( !interesting( path, fi.isDir() ) )
        return;

    if ( eventtype == KatWatcher::Delete ) {
        WatcherEvent e( path, KatIndexer::Delete );
        CHECKEVENT( e );
    } else if ( eventtype == KatWatcher::Move ) {
        //Add this to the to process list
        m_processMutex.lock();
        m_moveFiles[path] = oldpath;
        m_processMutex.unlock();

        WatcherEvent e( path, (KatIndexer::Action) (KatIndexer::Move|KatIndexer::Attrib) );
        m_eventMutex.lock();
        checkMoveEvent( e, oldpath );
        m_eventMutex.unlock();
    } else if ( eventtype == KatWatcher::Modify ) {
        WatcherEvent e( path, KatIndexer::Update );
        CHECKEVENT( e );
    } else if ( eventtype == KatWatcher::Create ) {
        if ( fi.isDir() )
            crawlOnMove ( path );
        else {
            WatcherEvent e( path, KatIndexer::Create );
            CHECKEVENT( e );
        }
    } else
        return;

    kdDebug( ddebug ) << k_funcinfo << " " << "Event Type " << eventtype <<  " occurred on file/dir " << path.latin1() << endl;

    if ( m_pause ) {
        if ( !m_committing )
            m_commitTimer.start( 120000, true );
    } else
        m_eventCond.wakeAll();
}

void KatIndexer::slotCommitTimeout()
{
    m_committing = true;

    // First process the waiting queue ..
    int hold = processWaiting ();

    if ( m_pause && !m_processFiles.empty() )
    {
        // FIXME - Ideally commitTimeout() should return quickly not hang around in here
        m_schedWait = m_scheduler->requestLock( this, m_catalogId, KatScheduler::Immediate );
        // Wait here for us to be scheduled
        if ( m_schedWait == -1 )
        {
            if ( parent() )
                QApplication::postEvent( parent(), new SubStatusEvent( m_catalogId, RequestSchedule) );
            m_schedCond.wait();
        }

        if ( !m_running || m_pause )
        {
            m_committing = false;
            // Release lock
            m_scheduler->releaseLock( m_catalogId, 0 );
            if ( parent() )
                QApplication::postEvent( parent(), new SubStatusEvent( m_catalogId, Reset ) );

            return;
        }

        // We have been scheduled
        if ( m_schedWait )
        {
            if ( parent() )
                QApplication::postEvent( parent(), new SubStatusEvent( m_catalogId, ScheduleTimeout, QVariant( m_schedWait ) ) );

            m_schedCond.wait( m_schedWait );

            if ( !m_running || m_pause )
            {
                // Release lock
                m_scheduler->releaseLock( m_catalogId, 0 );
                m_committing = false;
                if ( parent() )
                    QApplication::postEvent( parent(), new SubStatusEvent( m_catalogId, Reset ) );

                return;
            }
        }

        // Commit the events accumulated in m_processFiles to table
        m_processMutex.lock();
        QMap<QString, int> tempRecords = m_processFiles;
        m_processFiles.clear();
        m_processMutex.unlock();

        m_tempTable->addRecords( tempRecords );

        m_scheduler->releaseLock( m_catalogId, 0 );
    }

    if ( parent() )
        QApplication::postEvent( parent(), new SubStatusEvent( m_catalogId, Reset ) );

    if ( !m_waitingEvents.empty() )
    {
        if ( hold < 200 )
            m_commitTimer.start ( hold * 600, true );
        else
            m_commitTimer.start ( 120000, true );
    } else
        m_committing = false;
}

void KatIndexer::checkMoveEvent ( WatcherEvent newevent, QString oldpath )
{
    kdDebug( ddebug ) << "Checking move event " << endl;

    if ( newevent.m_action != KatIndexer::Move )
        return;

    // Take the last update/attrib event and queue it to dispatch before this one
    // because before we execute the move event, we want the file to be up-to-date

    // Find the oldpath
    // FIXME
    EventMap::Iterator it;
    for ( it = m_waitingEvents.begin(); it != m_waitingEvents.end(); ++it )
    {
        if ( it.key().startsWith( oldpath ) )
        {
            m_processMutex.lock();
            m_processFiles[QString( newevent.m_path ).append("/").append(it.key().latin1()+oldpath.length() + 1)] = it.data().m_action;
            m_processMutex.unlock();
            m_waitingEvents.remove( it );
        }
    }

    m_processMutex.lock();
    m_processFiles[newevent.m_path] = newevent.m_action;
    m_processMutex.unlock();
}

void KatIndexer::checkWatcherEvent ( WatcherEvent newevent )
{
    kdDebug( ddebug ) << "Checking wathcher event " << endl;

    // If the event is a Remove event -- remove corresponding event in m_waitingEvents and add it to dispatch Events
    if ( ( newevent.m_action == KatIndexer::Delete ) || ( newevent.m_action == KatIndexer::Create ) ) {
        m_waitingEvents.remove( newevent.m_path );
        m_processMutex.lock();
        m_processFiles[newevent.m_path] = newevent.m_action;
        m_processMutex.unlock();
    }

    // If it is a update event -- supplant previous event with this event.
    if ( newevent.m_action == KatIndexer::Update ) {
        EventMap::Iterator it = m_waitingEvents.find( newevent.m_path );
        if ( it != m_waitingEvents.end() )
            newevent.m_hold = it.data().m_hold + 1;

        newevent.m_time = QTime::currentTime().addMSecs( 512 );
        m_waitingEvents.insert( newevent.m_path, newevent );
    }

    // If it is a Attrib event -- find previous events if any and discard this in favor of those
    if ( newevent.m_action == KatIndexer::Attrib ) {
        EventMap::Iterator it = m_waitingEvents.find( newevent.m_path );

        if ( it != m_waitingEvents.end() ) {
            newevent.m_action = it.data().m_action;
            newevent.m_hold = it.data().m_hold + 1;
        }

        newevent.m_time = QTime::currentTime().addMSecs( 512 );
        m_waitingEvents.insert( newevent.m_path, newevent );
    }
}

bool KatIndexer::processFiles()
{
    QTime infoTime;
    QStringList infoWaiting;
    ActionMap sortedFiles;

    // Acquire lock before proceeding so that we can recover gracefully
    if ( !acquireLock( KatScheduler::Normal ) )
        return false;

    m_processMutex.lock();
    // First sort the files
    for ( ProcessMap::Iterator it = m_processFiles.begin(); it != m_processFiles.end() ; ++it ) {
        if ( it.data() & KatIndexer::Create )
            sortedFiles[ KatIndexer::Create ] += it.key();
        if ( it.data() & KatIndexer::Delete )
            sortedFiles[ KatIndexer::Delete ] += it.key();
        if ( it.data() & KatIndexer::Update )
            sortedFiles[ KatIndexer::Update ] += it.key();
        if ( it.data() & KatIndexer::Move )
            sortedFiles[ KatIndexer::Move ] += it.key();
        if ( it.data() & KatIndexer::Attrib )
            sortedFiles[ KatIndexer::Attrib ] += it.key();
    }
    m_processFiles.clear();
    m_processMutex.unlock();

    infoTime.start();

    // KatIndexer::Create - Newly created files, add to catalog
    if ( !sortedFiles[ KatIndexer::Create ].empty() )
    {
        if ( parent() )
        {
            QApplication::postEvent( parent(), new SubStatusEvent( m_catalogId, KatIndexer::AddFiles ) );
            QApplication::postEvent( parent(), new CurrentFileEvent( m_catalogId,
                                                                     sortedFiles[ KatIndexer::Create ].first(),
                                                                     sortedFiles[ KatIndexer::Create ].count() ) );
        }
        m_ksf->addFiles( sortedFiles[ KatIndexer::Create ] );
    }

    // KatIndexer::Delete - files removed from FS, remove from catalog
    if ( !sortedFiles[ KatIndexer::Delete ].empty() )
    {
        if ( parent() )
        {
            QApplication::postEvent( parent(), new SubStatusEvent( m_catalogId, KatIndexer::DeleteFiles ) );
            QApplication::postEvent( parent(), new CurrentFileEvent( m_catalogId,
                                                                     sortedFiles[ KatIndexer::Delete ].first(),
                                                                     sortedFiles[ KatIndexer::Delete ].count() ) );
        }
        m_ke->deleteFiles( m_catalog, sortedFiles[ KatIndexer::Delete ] );
    }

    // KatIndexer::Move - Move these files
    if ( !m_moveFiles.empty() )
    {
        // NOTE - If Q_ASSERT fires, should we prune m_moveFiles??
        Q_ASSERT( sortedFiles[ KatIndexer::Move ].count() == m_moveFiles.count() );

        if ( parent() )
        {
            QApplication::postEvent( parent(), new SubStatusEvent( m_catalogId, KatIndexer::MoveFiles ) );
            QApplication::postEvent( parent(), new CurrentFileEvent( m_catalogId,
                                                                     m_moveFiles.begin().key(),
                                                                     m_moveFiles.count() ) );
        }

        m_ksf->moveFiles( m_moveFiles );
    }

    // KatIndexer::Attrib -
    if ( !sortedFiles[ KatIndexer::Attrib ].empty() )
    {
        if ( parent() )
        {
            QApplication::postEvent( parent(), new SubStatusEvent( m_catalogId, KatIndexer::AttribFiles ) );
            QApplication::postEvent( parent(), new CurrentFileEvent( m_catalogId,
                                                                     sortedFiles[ KatIndexer::Attrib ].first(),
                                                                     sortedFiles[ KatIndexer::Attrib ].count() ) );
        }

        m_ksf->updateFiles( sortedFiles[ KatIndexer::Attrib ] );
    }

    // KatIndexer::Update - Update these
    infoWaiting = sortedFiles[ KatIndexer::Update ];

    sortedFiles.clear();
    m_moveFiles.clear();

    if ( !infoWaiting.empty() ) {

        // Discard old info saved from previous runs if we have not saved it
        // This happens when we sent a update request but did not wait for it to finish
        // and instead entered into pause/stop state. The state would have been saved
        // in KatInfoExtractor but is useless for us.
        m_kie->slotDiscardInfo();

        // FIXME -- fix this whole logic have a look at how KatInfoExtractor works
        // use a bool variable instead of checking whether to actually wait by observing if infoWaiting is empty
        // This way we can update files in this thread instead of using eventloop
        m_waiting = true;
        m_kie->updateFiles ( infoWaiting );

        kdDebug(ddebug) << "Found  " << infoWaiting.count() << " files in extract queue, dispatching .. " << endl;

        if ( m_waiting )
        {
            if ( parent() )
            {
                QApplication::postEvent( parent(), new SubStatusEvent( m_catalogId, KatIndexer::ExtractInfo ) );
                QApplication::postEvent( parent(), new CurrentFileEvent( m_catalogId,
                                                                         infoWaiting.first(),
                                                                         infoWaiting.count() ) );
            }

            m_infoCond.wait();

            kdDebug(ddebug) << "Woken up from waiting for extractor to finish" << endl;

            if ( !m_running )
            {
                // Release scheduler
                m_scheduler->releaseLock( m_catalogId, infoTime.elapsed() );
                return false;
            }

            if ( m_pause )
            {
                // Save the current state
                for ( QStringList::Iterator it = infoWaiting.begin(); it != infoWaiting.end(); ++it )
                    m_processFiles[*it] = KatIndexer::Update;
                infoWaiting.clear();

                m_scheduler->releaseLock( m_catalogId, infoTime.elapsed() );

                return false;
            }

            Q_ASSERT( !m_waiting );
        }

        kdDebug( ddebug ) << "Last extract info job containing  " << infoWaiting.count() << " files finished" << endl;

        m_kie->slotSaveInfo();

        if ( !m_running )
        {
            // Release scheduler
            m_scheduler->releaseLock( m_catalogId, infoTime.elapsed() );
            return false;
        }

        if ( m_pause )
        {
            // Save the current state
            for ( QStringList::Iterator it = infoWaiting.begin(); it != infoWaiting.end(); ++it )
                m_processFiles[*it] = KatIndexer::Update;
            infoWaiting.clear();

            m_scheduler->releaseLock( m_catalogId, infoTime.elapsed() );

            return false;
        }


        // Now fix up the Fileitems in the 'files' table
        m_ksf->updateFiles( infoWaiting );

        infoWaiting.clear();
    }

    // Update catalog
    m_ke->updateCatalog (m_catalog);

    // Reset status
    if ( parent() )
        QApplication::postEvent( parent(), new SubStatusEvent( m_catalogId, KatIndexer::Reset ) );

    m_scheduler->releaseLock( m_catalogId, infoTime.elapsed() );
    return true;
}

void KatIndexer::slotInfoExtractorCompleted()
{
    // we are either not running/we are paused ( have saved state )
    if ( !m_running || m_pause )
        return;

    m_waiting = false;
    m_infoCond.wakeAll();
}

bool KatIndexer::interesting( QStringList& files, bool isDir )
{
    const QStringList& searchIn = isDir ? m_ignoreDirs : m_ignoreFiles;
    QString pattern;
    QRegExp regexp;
    QStringList::Iterator it = files.begin();

    while ( it != files.end() )
    {
        // Go through all the files in list
        QStringList::ConstIterator si_it = searchIn.begin();
        for ( ;si_it != searchIn.end(); ++si_it )
        {

            if ( (*si_it).isEmpty() )
            {
                // ok, empty regexp. Ignoring it
                continue;
            }

            if ( (*si_it).startsWith("/") && (*si_it).endsWith("/") )
            {
                pattern = *si_it;
                pattern.remove( pattern.length() - 1, 1 ).remove(0, 1);
                regexp.setPattern( pattern );
                regexp.setWildcard( false );
            }
            else
            {
                regexp.setPattern( *si_it );
                regexp.setWildcard( true );
            }

            if ( regexp.search( *it ) != -1 )
            {
                kdDebug() << m_catalogId << " " << "Ignoring file/dir " << *it << " ( matches ignore file/dir pattern - " << *si_it << " )" << endl;
                it = files.remove( it );
                break;
            }
        }
        if ( si_it == searchIn.end() )
            ++it;
    }

    return ( !files.isEmpty() );
}

bool KatIndexer::interesting( const QString& file, bool isDir )
{
    const QStringList& searchIn = isDir ? m_ignoreDirs : m_ignoreFiles;
    QString pattern;
    QRegExp regexp;

    // Go through all the files in list
    QStringList::ConstIterator si_it = searchIn.begin();
    for ( ;si_it != searchIn.end(); ++si_it )
    {

        if ( (*si_it).isEmpty() )
        {
            // ok, empty regexp. Ignoring it
            continue;
        }

        if ( (*si_it).startsWith("/") && (*si_it).endsWith("/") )
        {
            pattern = *si_it;
            pattern.remove( pattern.length() - 1, 1 ).remove(0, 1);
            regexp.setPattern( pattern );
            regexp.setWildcard( false );
        }
        else
        {
            regexp.setPattern( *si_it );
            regexp.setWildcard( true );
        }

        if ( regexp.search( file ) != -1 )
        {
            kdDebug() << m_catalogId << " " << "Ignoring file/dir " << file << " ( matches ignore file/dir pattern - " << *si_it << " )" << endl;
            return false;
        }
    }

    return true;
}

#include "katindexer.moc"

Generated by  Doxygen 1.6.0   Back to index