Package com.dotcms.content.elasticsearch.business

Source Code of com.dotcms.content.elasticsearch.business.ESContentletIndexAPITest

package com.dotcms.content.elasticsearch.business;

import com.dotcms.TestBase;
import com.dotcms.content.elasticsearch.util.ESClient;
import com.dotcms.enterprise.publishing.sitesearch.SiteSearchResult;
import com.dotcms.enterprise.publishing.sitesearch.SiteSearchResults;
import com.dotmarketing.beans.ContainerStructure;
import com.dotmarketing.beans.Host;
import com.dotmarketing.beans.MultiTree;
import com.dotmarketing.beans.VersionInfo;
import com.dotmarketing.business.APILocator;
import com.dotmarketing.business.PermissionAPI;
import com.dotmarketing.cache.FieldsCache;
import com.dotmarketing.cache.StructureCache;
import com.dotmarketing.exception.DotDataException;
import com.dotmarketing.exception.DotSecurityException;
import com.dotmarketing.factories.MultiTreeFactory;
import com.dotmarketing.portlets.containers.model.Container;
import com.dotmarketing.portlets.contentlet.business.ContentletAPI;
import com.dotmarketing.portlets.contentlet.business.HostAPI;
import com.dotmarketing.portlets.contentlet.model.Contentlet;
import com.dotmarketing.portlets.folders.model.Folder;
import com.dotmarketing.portlets.htmlpages.model.HTMLPage;
import com.dotmarketing.portlets.languagesmanager.business.LanguageAPI;
import com.dotmarketing.portlets.languagesmanager.model.Language;
import com.dotmarketing.portlets.structure.factories.FieldFactory;
import com.dotmarketing.portlets.structure.factories.StructureFactory;
import com.dotmarketing.portlets.structure.model.Field;
import com.dotmarketing.portlets.structure.model.Structure;
import com.dotmarketing.portlets.templates.model.Template;
import com.dotmarketing.sitesearch.business.SiteSearchAPI;
import com.dotmarketing.util.Config;
import com.dotmarketing.util.Logger;
import com.dotmarketing.util.UUIDGenerator;
import com.dotmarketing.util.UtilMethods;
import com.liferay.portal.model.User;

import com.dotcms.repackage.org.elasticsearch.action.search.SearchPhaseExecutionException;
import com.dotcms.repackage.org.elasticsearch.action.search.SearchRequestBuilder;
import com.dotcms.repackage.org.elasticsearch.action.search.SearchResponse;
import com.dotcms.repackage.org.elasticsearch.client.Client;
import com.dotcms.repackage.org.elasticsearch.index.query.QueryBuilders;
import com.dotcms.repackage.org.elasticsearch.index.query.QueryStringQueryBuilder;
import com.dotcms.repackage.org.elasticsearch.search.SearchHits;
import com.dotcms.repackage.org.elasticsearch.search.internal.InternalSearchHits;
import com.dotcms.repackage.org.junit.BeforeClass;
import com.dotcms.repackage.org.junit.Ignore;
import com.dotcms.repackage.org.junit.Test;

import java.util.ArrayList;
import java.util.Date;
import java.util.List;

import static com.dotcms.repackage.org.junit.Assert.*;

/**
* @author Jonathan Gamba
*         Date: 4/18/13
*/
public class ESContentletIndexAPITest extends TestBase {

    private static String stemmerText;
    private static User user;
    private static Host defaultHost;
    private static String pageExt;
    private static Language defaultLanguage;

    @BeforeClass
    public static void prepare () throws DotSecurityException, DotDataException {

        HostAPI hostAPI = APILocator.getHostAPI();
        LanguageAPI languageAPI = APILocator.getLanguageAPI();

        //Setting the test user
        user = APILocator.getUserAPI().getSystemUser();
        defaultHost = hostAPI.findDefaultHost( user, false );
        pageExt = Config.getStringProperty( "VELOCITY_PAGE_EXTENSION" );
        //Getting the default language
        defaultLanguage = languageAPI.getDefaultLanguage();

        /*
        ORIGINAL TEXT
        A stemmer for English, for example, should identify the string 'cats' (and possibly 'catlike', 'catty' etc.) as based on the root 'cat',
        and 'stemmer', 'stemming', 'stemmed' as based on 'stem'.
        A stemming algorithm reduces the words 'fishing', 'fished', 'fish', and 'fisher' to the
        root word, 'fish'.
        On the other hand, 'argue', 'argued', 'argues', 'arguing', and 'argus' reduce to the stem 'argu' (illustrating the case where the stem is
        not itself a word or root) but 'argument' and 'arguments' reduce to the stem 'argument'.
         */
        //REMOVED ROOTS WORDS FOR TESTING
        stemmerText = "A stemmer for English, for example, should identify cat " +
                "and stemmer, stemming, stemmed. A stemming algorithm " +
                "reduces the words fishing, fished, and fisher to the " +
                "root word... On the other hand, argue, argued, argues, arguing, " +
                "and argus reduce to the stem ... (illustrating the case where the stem is " +
                "not itself a word or root) but arguments reduce to the stem .....";
    }

    /**
     * Testing the {@link ContentletIndexAPI#createContentIndex(String)}, {@link ContentletIndexAPI#delete(String)} and
     * {@link ContentletIndexAPI#listDotCMSIndices()} methods
     *
     * @throws Exception
     * @see ContentletIndexAPI
     * @see ESContentletIndexAPI
     */
    @Test
    public void createContentIndexAndDelete () throws Exception {

        ContentletIndexAPI indexAPI = APILocator.getContentletIndexAPI();

        //Build the index names
        String timeStamp = String.valueOf( new Date().getTime() );
        String workingIndex = ESContentletIndexAPI.ES_WORKING_INDEX_NAME + "_" + timeStamp;
        String liveIndex = ESContentletIndexAPI.ES_LIVE_INDEX_NAME + "_" + timeStamp;

        //Get all the indices
        List<String> indices = indexAPI.listDotCMSIndices();
        //Validate
        assertNotNull( indices );
        assertTrue( !indices.isEmpty() );
        int oldIndices = indices.size();

        //Creates the working index
        Boolean result = indexAPI.createContentIndex( workingIndex );
        //Validate
        assertTrue( result );

        //Creates the live index
        result = indexAPI.createContentIndex( liveIndex );
        //Validate
        assertTrue( result );

        //***************************************************
        //Get all the indices
        indices = indexAPI.listDotCMSIndices();
        //Validate
        assertNotNull( indices );
        assertTrue( !indices.isEmpty() );
        int newIndices = indices.size();

        //Search for the just saved indices
        Boolean foundWorking = false;
        Boolean foundLive = false;
        for ( String index : indices ) {
            if ( index.equals( liveIndex ) ) {
                foundLive = true;
            } else if ( index.equals( workingIndex ) ) {
                foundWorking = true;
            }
        }
        //Validate
        assertTrue( foundWorking );
        assertTrue( foundLive );

        //Verify we just added two more indices
        assertTrue( oldIndices + 2 == newIndices );

        //***************************************************
        //Now lets delete the created indices
        Boolean deleted = indexAPI.delete( workingIndex );
        assertTrue( deleted );
        deleted = indexAPI.delete( liveIndex );
        assertTrue( deleted );

        //***************************************************
        //Get all the indices again....
        indices = indexAPI.listDotCMSIndices();
        //Validate
        assertNotNull( indices );
        assertTrue( !indices.isEmpty() );
        newIndices = indices.size();

        //Verify if we still find the deleted indices
        foundWorking = false;
        foundLive = false;
        for ( String index : indices ) {
            if ( index.equals( liveIndex ) ) {
                foundLive = true;
            } else if ( index.equals( workingIndex ) ) {
                foundWorking = true;
            }
        }
        //Validate
        assertFalse( foundWorking );
        assertFalse( foundLive );

        //Verify we just added two more indices
        assertTrue( oldIndices == newIndices );
    }

    /**
     * Testing the {@link ContentletIndexAPI#activateIndex(String)}, {@link ContentletIndexAPI#deactivateIndex(String)}
     * and {@link ContentletIndexAPI#getCurrentIndex()} methods
     *
     * @throws Exception
     * @see ContentletIndexAPI
     * @see ESContentletIndexAPI
     */
    @Test
    @Ignore
    public void activateDeactivateIndex () throws Exception {

        ContentletIndexAPI indexAPI = APILocator.getContentletIndexAPI();

        //Build the index names
        String timeStamp = String.valueOf( new Date().getTime() );
        String workingIndex = ESContentletIndexAPI.ES_WORKING_INDEX_NAME + "_" + timeStamp;
        String liveIndex = ESContentletIndexAPI.ES_LIVE_INDEX_NAME + "_" + timeStamp;

        String oldActiveLive = indexAPI.getActiveIndexName(ContentletIndexAPI.ES_LIVE_INDEX_NAME);
        String oldActiveWorking = indexAPI.getActiveIndexName(ContentletIndexAPI.ES_WORKING_INDEX_NAME);

        //Creates the working index
        Boolean result = indexAPI.createContentIndex( workingIndex );
        assertTrue( result );
        //Activate this working index
        indexAPI.activateIndex( workingIndex );

        //Creates the live index
        result = indexAPI.createContentIndex( liveIndex );
        assertTrue( result );
        //Activate this live index
        indexAPI.activateIndex( liveIndex );

        //***************************************************
        //Get the current indices
        String liveActiveIndex = indexAPI.getActiveIndexName( ContentletIndexAPI.ES_LIVE_INDEX_NAME );

        //Validate
        assertNotNull( liveActiveIndex );
        assertEquals( liveActiveIndex, liveIndex );

        //***************************************************
        //Now lets deactivate the indices
        //Deactivate this working index
        indexAPI.deactivateIndex( workingIndex );
        //Deactivate this live index
        indexAPI.deactivateIndex( liveIndex );

        // restore old active index
        indexAPI.activateIndex(oldActiveWorking);
        indexAPI.activateIndex(oldActiveLive);
    }

    /**
     * Testing {@link ContentletIndexAPI#isDotCMSIndexName(String)}
     *
     * @throws Exception
     * @see ContentletIndexAPI
     * @see ESContentletIndexAPI
     */
    @Test
    public void isDotCMSIndexName () throws Exception {

        ContentletIndexAPI indexAPI = APILocator.getContentletIndexAPI();

        //Build the index names
        String timeStamp = String.valueOf( new Date().getTime() );
        String workingIndex = ESContentletIndexAPI.ES_WORKING_INDEX_NAME + "_" + timeStamp;

        //Verify with a proper name
        boolean isIndexName = indexAPI.isDotCMSIndexName( workingIndex );
        assertTrue( isIndexName );

        //Verify a non proper name
        workingIndex = "TEST" + "_" + timeStamp;
        isIndexName = indexAPI.isDotCMSIndexName( workingIndex );
        assertFalse( isIndexName );
    }

    /**
     * Testing {@link ContentletIndexAPI#optimize(java.util.List)}
     *
     * @throws Exception
     * @see ContentletIndexAPI
     * @see ESContentletIndexAPI
     */
    @Test
    public void optimize () throws Exception {

        ContentletIndexAPI indexAPI = APILocator.getContentletIndexAPI();

        //Build the index names
        String timeStamp = String.valueOf( new Date().getTime() );
        String workingIndex = ESContentletIndexAPI.ES_WORKING_INDEX_NAME + "_" + timeStamp;
        String liveIndex = ESContentletIndexAPI.ES_LIVE_INDEX_NAME + "_" + timeStamp;

        //Creates the working index
        Boolean result = indexAPI.createContentIndex( workingIndex );
        //Validate
        assertTrue( result );

        //Creates the live index
        result = indexAPI.createContentIndex( liveIndex );
        //Validate
        assertTrue( result );

        //Test the optimize method
        List<String> indices = new ArrayList<String>();
        indices.add( workingIndex );
        indices.add( liveIndex );
        Boolean optimized = indexAPI.optimize( indices );
        //Validate
        assertTrue( optimized );
    }

    /**
     * Testing {@link ContentletIndexAPI#getRidOfOldIndex()}
     *
     * @throws Exception
     * @see ContentletIndexAPI
     * @see ESContentletIndexAPI
     */
    @Test
    public void getRidOfOldIndex () throws Exception {
        /*
        ContentletIndexAPI indexAPI = APILocator.getContentletIndexAPI();

        List<String> oldIndices = indexAPI.getCurrentIndex();
        //Validate
        assertNotNull( oldIndices );
        assertTrue( !oldIndices.isEmpty() );

        indexAPI.getRidOfOldIndex();

        List<String> indices = indexAPI.getCurrentIndex();
        //Validate
        assertNotNull( indices );
        assertTrue( !indices.isEmpty() );

        assertNotSame( oldIndices, indices );

        indices = indexAPI.getNewIndex();
        //Validate
        assertNotNull( indices );
        assertTrue( indices.isEmpty() );
        */

        // this blows up everything. do not try at home
    }

    /**
     * Testing the {@link ContentletIndexAPI#addContentToIndex(com.dotmarketing.portlets.contentlet.model.Contentlet)},
     * {@link ContentletIndexAPI#removeContentFromIndex(com.dotmarketing.portlets.contentlet.model.Contentlet)} methods
     *
     * @throws Exception
     * @see ContentletAPI
     * @see ContentletIndexAPI
     * @see ESContentletIndexAPI
     */
    @Test
    public void addRemoveContentToIndex () throws Exception {

        ContentletIndexAPI indexAPI = APILocator.getContentletIndexAPI();
        ContentletAPI contentletAPI = APILocator.getContentletAPI();

        //Creating a test structure
        Structure testStructure = loadTestStructure();
        //Creating a test contentlet
        Contentlet testContentlet = loadTestContentlet( testStructure );

        try {

            //And add it to the index
            indexAPI.addContentToIndex( testContentlet );

            //We are just making time in order to let it apply the index
            contentletAPI.isInodeIndexed( testContentlet.getInode(), true );

            //Verify if it was added to the index
            String query = "+structureName:" + testStructure.getVelocityVarName() + " +deleted:false +live:true";
            List<Contentlet> result = contentletAPI.search( query, 0, -1, "modDate desc", user, true );

            //Validations
            assertNotNull( result );
            assertTrue( !result.isEmpty() );

            //Remove the contentlet from the index
            indexAPI.removeContentFromIndex( testContentlet );

            //We are just making time in order to let it apply the index
            wasContentRemoved( query );

            //Verify if it was removed to the index
            result = contentletAPI.search( query, 0, -1, "modDate desc", user, true );

            //Validations
            assertTrue( result == null || result.isEmpty() );
        } finally {
            APILocator.getContentletAPI().delete( testContentlet, user, false );
        }
    }

    /**
     * Testing {@link ContentletIndexAPI#removeContentFromIndexByStructureInode(String)}
     *
     * @throws Exception
     * @see ContentletAPI
     * @see ContentletIndexAPI
     * @see ESContentletIndexAPI
     */
    @Test
    public void removeContentFromIndexByStructureInode () throws Exception {

        ContentletIndexAPI indexAPI = APILocator.getContentletIndexAPI();
        ContentletAPI contentletAPI = APILocator.getContentletAPI();

        //Creating a test structure
        Structure testStructure = loadTestStructure();
        //Creating a test contentlet
        Contentlet testContentlet = loadTestContentlet( testStructure );

        try {
            //And add it to the index
            indexAPI.addContentToIndex( testContentlet );

            //We are just making time in order to let it apply the index
            contentletAPI.isInodeIndexed( testContentlet.getInode(), true );

            //Verify if it was added to the index
            String query = "+structureName:" + testStructure.getVelocityVarName() + " +deleted:false +live:true";
            List<Contentlet> result = contentletAPI.search( query, 0, -1, "modDate desc", user, true );

            //Validations
            assertNotNull( result );
            assertTrue( !result.isEmpty() );

            //Remove the contentlet from the index
            indexAPI.removeContentFromIndexByStructureInode( testStructure.getInode() );

            int x=0;
            do {
                Thread.sleep(200);
                //Verify if it was removed to the index
                result = contentletAPI.search( query, 0, -1, "modDate desc", user, true );
                x++;
            } while((result == null || result.isEmpty()) && x<100);

        } finally {
            APILocator.getContentletAPI().delete( testContentlet, user, false );
        }
    }

    /**
     * Testing the {@link SiteSearchAPI#putToIndex(String, com.dotcms.enterprise.publishing.sitesearch.SiteSearchResult)},
     * {@link SiteSearchAPI#search(String, String, int, int)} and {@link SiteSearchAPI#deleteFromIndex(String, String)} methods
     *
     * @throws Exception
     * @see ContentletAPI
     * @see ContentletIndexAPI
     * @see ESContentletIndexAPI
     */
    @Test
    public void testSearch () throws Exception {

        SiteSearchAPI siteSearchAPI = APILocator.getSiteSearchAPI();

        //*****************************************************************
        //Verify if we already have and site search index, if not lets create one...
        IndiciesAPI.IndiciesInfo indiciesInfo = APILocator.getIndiciesAPI().loadIndicies();
        String currentSiteSearchIndex = indiciesInfo.site_search;
        String indexName = currentSiteSearchIndex;
        if ( currentSiteSearchIndex == null ) {
            indexName = SiteSearchAPI.ES_SITE_SEARCH_NAME + "_" + ESContentletIndexAPI.timestampFormatter.format( new Date() );
            APILocator.getSiteSearchAPI().createSiteSearchIndex( indexName, null, 1 );
            APILocator.getSiteSearchAPI().activateIndex( indexName );
        }

        //*****************************************************************
        //Creating a test structure
        Structure testStructure = loadTestStructure();
        //Creating a test contentlet
        Contentlet testContentlet = loadTestContentlet( testStructure );
        //Creating a test html page
        HTMLPage testHtmlPage = loadHtmlPage( testContentlet );

        //*****************************************************************
        //Build a site search result in order to add it to the index
        VersionInfo versionInfo = APILocator.getVersionableAPI().getVersionInfo( testHtmlPage.getIdentifier() );
        String docId = testHtmlPage.getIdentifier() + "_" + defaultLanguage.getId();

        SiteSearchResult res = new SiteSearchResult( testHtmlPage.getMap() );
        res.setLanguage( defaultLanguage.getId() );
        res.setFileName( testHtmlPage.getFriendlyName() );
        res.setModified( versionInfo.getVersionTs() );
        res.setHost( defaultHost.getIdentifier() );
        res.setMimeType( "text/html" );
        res.setContentLength( 1 );//Just sending something different than 0
        res.setContent( stemmerText );
        res.setUri( testHtmlPage.getURI() );
        res.setUrl( testHtmlPage.getPageUrl() );
        res.setId( docId );

        //Adding it to the index
        siteSearchAPI.putToIndex( indexName, res );
        isDocIndexed( docId );

        try {

            //++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
            //+++++++++++++++++++++++++STEMMERS+++++++++++++++++++++++++
            /*
            NOTE: THE CONTENT TEXT DOES NOT CONTAIN THE ROOT WORDS, THIS IS JUST THE REFERENCE TEXT SHOWING HOW SHOULD WORKS!!

            A stemmer for English, for example, should identify the string 'cats' (and possibly 'catlike', 'catty' etc.) as based on the root 'cat',
            and 'stemmer', 'stemming', 'stemmed' as based on 'stem'.
            A stemming algorithm reduces the words 'fishing', 'fished', 'fish', and 'fisher' to the
            root word, 'fish'.
            On the other hand, 'argue', 'argued', 'argues', 'arguing', and 'argus' reduce to the stem 'argu' (illustrating the case where the stem is
            not itself a word or root) but 'argument' and 'arguments' reduce to the stem 'argument'.

            NOTE: THE CONTENT TEXT DOES NOT CONTAIN THE ROOT WORDS, THIS IS JUST THE REFERENCE TEXT SHOWING HOW SHOULD WORKS!!
             */

            //Testing the stemer
            SiteSearchResults siteSearchResults = siteSearchAPI.search( indexName, "argu", 0, 100 );
            //Validations
            assertTrue( siteSearchResults.getError() == null || siteSearchResults.getError().isEmpty() );
            assertTrue( siteSearchResults.getTotalResults() > 0 );
            String highLights = siteSearchResults.getResults().get( 0 ).getHighLights()[0];
            assertTrue( highLights.contains( "<em>argue</em>" ) );
            assertTrue( highLights.contains( "<em>argued</em>" ) );
            assertTrue( highLights.contains( "<em>argues</em>" ) );
            assertTrue( highLights.contains( "<em>arguing</em>" ) );
            //assertTrue( highLights.contains( "<em>argus</em>" ) );//Not found..., verify this....

            //Testing the stemer
            siteSearchResults = siteSearchAPI.search( indexName, "cats", 0, 100 );
            //Validations
            assertTrue( siteSearchResults.getError() == null || siteSearchResults.getError().isEmpty() );
            assertTrue( siteSearchResults.getTotalResults() > 0 );
            highLights = siteSearchResults.getResults().get( 0 ).getHighLights()[0];
            assertTrue( highLights.contains( "<em>cat</em>" ) );

            //Testing the stemer
            siteSearchResults = siteSearchAPI.search( indexName, "stem", 0, 100 );
            //Validations
            assertTrue( siteSearchResults.getError() == null || siteSearchResults.getError().isEmpty() );
            assertTrue( siteSearchResults.getTotalResults() > 0 );
            highLights = siteSearchResults.getResults().get( 0 ).getHighLights()[0];
            //assertTrue( highLights.contains( "<em>stemmer</em>" ) );//Not found..., verify this....
            assertTrue( highLights.contains( "<em>stemming</em>" ) );
            assertTrue( highLights.contains( "<em>stemmed</em>" ) );

            //Testing the stemer
            siteSearchResults = siteSearchAPI.search( indexName, "argument", 0, 100 );
            //Validations
            assertTrue( siteSearchResults.getError() == null || siteSearchResults.getError().isEmpty() );
            assertTrue( siteSearchResults.getTotalResults() > 0 );
            highLights = siteSearchResults.getResults().get( 0 ).getHighLights()[0];
            assertTrue( highLights.contains( "<em>arguments</em>" ) );

            //++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
            //++++++++++++++++++++++++++NGrams++++++++++++++++++++++++++
            //Testing the search with words that are in each extreme of the text
            siteSearchResults = siteSearchAPI.search( indexName, "english illustrating", 0, 100 );
            //Validations
            assertTrue( siteSearchResults.getError() == null || siteSearchResults.getError().isEmpty() );
            assertTrue( siteSearchResults.getTotalResults() > 0 );

            //Testing the search with words that are not in find order
            siteSearchResults = siteSearchAPI.search( indexName, "arguments algorithm", 0, 100 );
            //Validations
            assertTrue( siteSearchResults.getError() == null || siteSearchResults.getError().isEmpty() );
            assertTrue( siteSearchResults.getTotalResults() > 0 );

            //++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
            //++++++++++++++++++++++++OTHER TESTS+++++++++++++++++++++++
            //Testing the search
            siteSearchResults = siteSearchAPI.search( indexName, "engli*", 0, 100 );
            //Validations
            assertTrue( siteSearchResults.getError() == null || siteSearchResults.getError().isEmpty() );
            assertTrue( siteSearchResults.getTotalResults() > 0 );

            //Testing the search
            siteSearchResults = siteSearchAPI.search( indexName, "*engli", 0, 100 );
            //Validations
            assertTrue( siteSearchResults.getError() == null || siteSearchResults.getError().isEmpty() );
            assertEquals( siteSearchResults.getTotalResults(), 0 );

            //Testing the search
            siteSearchResults = siteSearchAPI.search( indexName, "e", 0, 100 );
            //Validations
            assertTrue( siteSearchResults.getError() == null || siteSearchResults.getError().isEmpty() );
            assertEquals( siteSearchResults.getTotalResults(), 0 );

            //Testing the search with a non existing word
            siteSearchResults = siteSearchAPI.search( indexName, "weird", 0, 100 );
            //Validations
            assertTrue( siteSearchResults.getError() == null || siteSearchResults.getError().isEmpty() );
            assertEquals( siteSearchResults.getTotalResults(), 0 );
        } finally {
            //And finally remove the index
            siteSearchAPI.deleteFromIndex( indexName, docId );
        }
    }

    @Test
    public void testSearchIndexByDate () throws Exception {

      Folder testFolder = null;
      Structure testStructure = null;
      Contentlet testContent = null;
      ContentletAPI contentletAPI = APILocator.getContentletAPI();

      try {
        ContentletIndexAPI indexAPI = APILocator.getContentletIndexAPI();

        //Creating a test structure
        PermissionAPI permissionAPI = APILocator.getPermissionAPI();

        //Set up a test folder
        testFolder = APILocator.getFolderAPI().createFolders( "testSearchIndexByDateFolder", defaultHost, user, false );
        permissionAPI.permissionIndividually( permissionAPI.findParentPermissionable( testFolder ), testFolder, user, false );

        //Set up a test structure
        String structureName = "testSearchIndexByDateStructure";
        testStructure = new Structure();
        testStructure.setHost( defaultHost.getIdentifier() );
        testStructure.setFolder( testFolder.getInode() );
        testStructure.setName( structureName );
        testStructure.setStructureType( Structure.STRUCTURE_TYPE_CONTENT );
        testStructure.setOwner( user.getUserId() );
        testStructure.setVelocityVarName( structureName );
        StructureFactory.saveStructure( testStructure );
        StructureCache.addStructure( testStructure );
        //Adding test field

        Field field = new Field( "testSearchIndexByDateField", Field.FieldType.DATE_TIME,  Field.DataType.DATE, testStructure, true, true, true, 1, false, false, true );
        field.setVelocityVarName( "testSearchIndexByDateField" );
        field.setListed( true );
        FieldFactory.saveField( field );
        FieldsCache.addField( field );

        //Creating a test contentlet
        testContent = new Contentlet();
        testContent.setReviewInterval( "1m" );
        testContent.setStructureInode( testStructure.getInode() );
        testContent.setHost( defaultHost.getIdentifier() );
        testContent.setLanguageId(1);

        contentletAPI.setContentletProperty( testContent, field, "03/05/2014" );

        testContent = contentletAPI.checkin( testContent, null, permissionAPI.getPermissions( testStructure ), user, false );
        APILocator.getVersionableAPI().setLive(testContent);

      //And add it to the index
      indexAPI.addContentToIndex( testContent );

      //We are just making time in order to let it apply the index
      contentletAPI.isInodeIndexed( testContent.getInode(), true );

      //Verify if it was added to the index
      String query = "+structureName:" + testStructure.getVelocityVarName() + " +testSearchIndexByDateStructure.testSearchIndexByDateField:03/05/2014 +deleted:false +live:true";
      List<Contentlet> result = contentletAPI.search( query, 0, -1, "modDate desc", user, true );

      assertTrue(UtilMethods.isSet(result) && !result.isEmpty());

      } catch(Exception e) {
        Logger.error("Error executing testSearchIndexByDate", e.getMessage(), e);
      } finally {
        if(UtilMethods.isSet(testStructure)) {
          APILocator.getStructureAPI().delete(testStructure, user);
        }
        if(UtilMethods.isSet(testFolder)) {
          APILocator.getFolderAPI().delete(testFolder, user, false);
        }
      }

    }

    /**
     * Creates and returns a test html page
     *
     * @param contentlet
     * @throws DotSecurityException
     * @throws DotDataException
     */
    private HTMLPage loadHtmlPage ( Contentlet contentlet ) throws DotSecurityException, DotDataException {

        Structure structure = contentlet.getStructure();

        //Create a container for the given contentlet
        Container container = new Container();
        container.setCode( "this is the code" );
        container.setFriendlyName( "test container" );
        container.setTitle( "his is the title" );
        container.setMaxContentlets( 5 );
        container.setPreLoop( "preloop code" );
        container.setPostLoop( "postloop code" );
        //Save it

        List<ContainerStructure> csList = new ArrayList<ContainerStructure>();
        ContainerStructure cs = new ContainerStructure();
        cs.setStructureId(structure.getInode());
        cs.setCode("this is the code");
        csList.add(cs);

        container = APILocator.getContainerAPI().save( container, csList, defaultHost, user, false );

        //Create a template
        String body = "<html><body> #parseContainer('" + container.getIdentifier() + "') </body></html>";
        String title = "empty test template " + UUIDGenerator.generateUuid();

        Template template = new Template();
        template.setTitle( title );
        template.setBody( body );

        template = APILocator.getTemplateAPI().saveTemplate( template, defaultHost, user, false );

        //Create the html page
        String pageUrl = "testpage_" + new Date().getTime() + "." + pageExt;
        Folder homeFolder = APILocator.getFolderAPI().createFolders( "/home/", defaultHost, user, false );
        HTMLPage htmlPage = new HTMLPage();
        htmlPage.setPageUrl( pageUrl );
        htmlPage.setFriendlyName( pageUrl );
        htmlPage.setTitle( pageUrl );
        htmlPage = APILocator.getHTMLPageAPI().saveHTMLPage( htmlPage, template, homeFolder, user, false );
        //Make it live
        APILocator.getVersionableAPI().setLive( htmlPage );

        MultiTree multiTree = new MultiTree();
        multiTree.setParent1( htmlPage.getIdentifier() );
        multiTree.setParent2( container.getIdentifier() );
        multiTree.setChild( contentlet.getIdentifier() );
        multiTree.setTreeOrder( 1 );
        MultiTreeFactory.saveMultiTree( multiTree );

        return htmlPage;
    }

    /**
     * Creates and returns a test Structure
     *
     * @return
     * @throws Exception
     */
    private Structure loadTestStructure () throws Exception {

        PermissionAPI permissionAPI = APILocator.getPermissionAPI();

        //Set up a test folder
        Folder testFolder = APILocator.getFolderAPI().createFolders( "/" + new Date().getTime() + "/", defaultHost, user, false );
        permissionAPI.permissionIndividually( permissionAPI.findParentPermissionable( testFolder ), testFolder, user, false );

        //Set up a test structure
        String structureName = "ESContentletIndexAPITest_" + new Date().getTime();
        Structure testStructure = new Structure();
        testStructure.setHost( defaultHost.getIdentifier() );
        testStructure.setFolder( testFolder.getInode() );
        testStructure.setName( structureName );
        testStructure.setStructureType( Structure.STRUCTURE_TYPE_CONTENT );
        testStructure.setOwner( user.getUserId() );
        testStructure.setVelocityVarName( structureName );
        StructureFactory.saveStructure( testStructure );
        StructureCache.addStructure( testStructure );
        //Adding test field
        Field field = new Field( "Wysiwyg", Field.FieldType.WYSIWYG, Field.DataType.LONG_TEXT, testStructure, true, true, true, 1, false, false, false );
        field.setVelocityVarName( "wysiwyg" );
        field.setListed( true );
        FieldFactory.saveField( field );
        FieldsCache.addField( field );

        return testStructure;
    }

    /**
     * Creates and returns a test Contentlet
     *
     * @return
     * @throws Exception
     */
    private Contentlet loadTestContentlet ( Structure testStructure ) throws Exception {

        //Set up a test contentlet
        Contentlet testContentlet = new Contentlet();
        testContentlet.setStructureInode( testStructure.getInode() );
        testContentlet.setHost( defaultHost.getIdentifier() );
        testContentlet.setLanguageId( defaultLanguage.getId() );
        testContentlet.setStringProperty( "wysiwyg", stemmerText );
        testContentlet = APILocator.getContentletAPI().checkin( testContentlet, user, false );

        //Make it live
        APILocator.getVersionableAPI().setLive( testContentlet );

        return testContentlet;
    }

    private boolean isDocIndexed ( String id ) {

        if ( !UtilMethods.isSet( id ) ) {
            Logger.warn( this, "Requested Inode is not indexed because Inode is not set" );
        }
        SearchHits lc;
        boolean found = false;
        int counter = 0;
        while ( counter < 300 ) {
            try {
                lc = indexSearch( "+id:" + id, "modDate" );
            } catch ( Exception e ) {
                Logger.error( this.getClass(), e.getMessage(), e );
                return false;
            }
            if ( lc.getTotalHits() > 0 ) {
                found = true;
                return true;
            }
            try {
                Thread.sleep( 100 );
            } catch ( Exception e ) {
                Logger.debug( this, "Cannot sleep : ", e );
            }
            counter++;
        }
        return found;
    }

    private SearchHits indexSearch ( String query, String sortBy ) throws Exception {

        String qq = ESContentFactoryImpl.translateQuery( query, sortBy ).getQuery();

        // we check the query to figure out wich indexes to hit
        String indexToHit;
        IndiciesAPI.IndiciesInfo info;
        try {
            info = APILocator.getIndiciesAPI().loadIndicies();
        } catch ( DotDataException ee ) {
            Logger.fatal( this, "Can't get indicies information", ee );
            return null;
        }
        indexToHit = info.site_search;
        String indexName = indexToHit;
        if ( indexName == null ) {
            indexName = SiteSearchAPI.ES_SITE_SEARCH_NAME + "_" + ESContentletIndexAPI.timestampFormatter.format( new Date() );
            APILocator.getSiteSearchAPI().createSiteSearchIndex( indexName, null, 1 );
            APILocator.getSiteSearchAPI().activateIndex( indexName );
        }

        Client client = new ESClient().getClient();
        SearchResponse resp;
        try {
            QueryStringQueryBuilder qb = QueryBuilders.queryString( qq );
            SearchRequestBuilder srb = client.prepareSearch();
            srb.setQuery( qb );

            srb.setIndices( indexName );
            srb.addFields( "id" );

            try {
                resp = srb.execute().actionGet();
            } catch ( SearchPhaseExecutionException e ) {
                if ( e.getMessage().contains( "dotraw] in order to sort on" ) ) {
                    return new InternalSearchHits( InternalSearchHits.EMPTY, 0, 0 );
                } else {
                    throw e;
                }
            }
        } catch ( Exception e ) {
            Logger.error( ESContentFactoryImpl.class, e.getMessage(), e );
            throw new RuntimeException( e );
        }
        return resp.getHits();
    }

    public boolean wasContentRemoved ( String query ) {

        ContentletAPI contentletAPI = APILocator.getContentletAPI();

        boolean removed = false;
        int counter = 0;
        while ( counter < 300 ) {
            try {
                //Verify if it was removed to the index
                List<Contentlet> result = contentletAPI.search( query, 0, -1, "modDate desc", user, true );

                //Validations
                if ( result == null || result.isEmpty() ) {
                    return true;
                }
            } catch ( Exception e ) {
                Logger.error( this.getClass(), e.getMessage(), e );
                return false;
            }

            try {
                Thread.sleep( 100 );
            } catch ( Exception e ) {
                Logger.debug( this, "Cannot sleep : ", e );
            }
            counter++;
        }

        return removed;
    }

}
TOP

Related Classes of com.dotcms.content.elasticsearch.business.ESContentletIndexAPITest

TOP
Copyright © 2018 www.massapi.com. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.