Package ivory.smrf.model.builder

Examples of ivory.smrf.model.builder.MRFBuilder

122
123
124
125
126
127
128
129
130
131
132
      Node modelNode = models.get(modelID);
      Node expanderNode = expanders.get(modelID);

      // Initialize retrieval environment variables.
      QueryRunner runner = null;
      MRFBuilder builder = null;
      MRFExpander expander = null;
      try {
        // Get the MRF builder.
        builder = MRFBuilder.get(env, modelNode.cloneNode(true));
View Full Code Here
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
      // compute features for each model
      for(String modelName : modelNames) {
        // build mrf from model node
        Node modelNode = runner.getModel(modelName);
        MRFBuilder builder = MRFBuilder.get(env, modelNode);     
        MarkovRandomField mrf = builder.buildMRF(env.tokenize(queryText));

        // get mrf cliques
        List<Clique> cliques = mrf.getCliques();

        // add parameter name to feature name set
        for(Clique c : cliques) {
          // parameter id
          String paramId = c.getParameter().getName();

          // handle linear importance model weights
          if(importanceModels.size() != 0) {
            for(LinearImportanceModel model : linearImportanceModels) {
              List<MetaFeature> metaFeatures = model.getMetaFeatures();

              for(MetaFeature metaFeat : metaFeatures) {
                // feature id = modelName-metaFeatId-paramId
                String featId = modelName + "-" + metaFeat.getName() + "-" + paramId;
                featureNames.add(featId);
              }
            }
          }

          // feature id = modelName-paramId
          String featId = modelName + "-" + paramId;

          featureNames.add(featId);
        }
      }     
    }

    // add judgment feature name
    featureNames.add(JUDGMENT_FEATURE_NAME);

    // print feature name header
    System.out.print(QUERY_FEATURE_NAME + "\t" + DOC_FEATURE_NAME);
    for(String featureName : featureNames) {
      System.out.print("\t" + featureName);
    }
    System.out.println();

    // extract features query-by-query
    for(Entry<String, String> queryEntry : queries.entrySet()) {
      // feature map (docname -> feature name -> feature value)
      SortedMap<String,SortedMap<String,Double>> featureValues = new TreeMap<String,SortedMap<String,Double>>();

      // query id and text
      String qid = queryEntry.getKey();
      String queryText = queryEntry.getValue();

      // compute features for each model
      for(String modelName : modelNames) {
        // build mrf from model node
        Node modelNode = runner.getModel(modelName);
        MRFBuilder builder = MRFBuilder.get(env, modelNode);     
        MarkovRandomField mrf = builder.buildMRF(env.tokenize(queryText));

        // initialize mrf
        mrf.initialize();

        // get mrf cliques
View Full Code Here
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
     // compute features for each model
     for(String modelName : modelNames) {
       // build mrf from model node
       Node modelNode = runner.getModel(modelName);
       MRFBuilder builder = MRFBuilder.get(env, modelNode);
       MarkovRandomField mrf = builder.buildMRF(queryText.split("\\s+"));

       // get mrf cliques
      List<Clique> cliques = mrf.getCliques();

      // add parameter name to feature name set
      for(Clique c : cliques) {
        // parameter id
        String paramId = c.getParameter().getName();

        // handle linear importance model weights
        if(importanceModels.size() != 0) {
          for(LinearImportanceModel model : linearImportanceModels) {
            List<MetaFeature> metaFeatures = model.getMetaFeatures();

            for(MetaFeature metaFeat : metaFeatures) {
              // feature id = modelName-metaFeatId-paramId
              String featId = modelName + "-" + metaFeat.getName() + "-" + paramId;
              featureNames.add(featId);
            }
          }
        }

        // feature id = modelName-paramId
        String featId = modelName + "-" + paramId;

        featureNames.add(featId);
      }
     }
   }

   // add judgment feature name
   featureNames.add(JUDGMENT_FEATURE_NAME);

   // print feature name header
   System.out.print(QUERY_FEATURE_NAME + "\t" + DOC_FEATURE_NAME);
   for(String featureName : featureNames) {
     System.out.print("\t" + featureName);
   }
   System.out.println();

   // extract features query-by-query
   for(Entry<String, String> queryEntry : queries.entrySet()) {
     // feature map (docname -> feature name -> feature value)
     SortedMap<String,SortedMap<String,Operator>> featureValues = new TreeMap<String,SortedMap<String,Operator>>();

     // query id and text
     String qid = queryEntry.getKey();
     String queryText = queryEntry.getValue();

     // compute features for each model
     for(String modelName : modelNames) {
       // build mrf from model node
       Node modelNode = runner.getModel(modelName);
       MRFBuilder builder = MRFBuilder.get(env, modelNode);
       MarkovRandomField mrf = builder.buildMRF(queryText.split("\\s+"));

       // initialize mrf
       mrf.initialize();

       // get mrf cliques
View Full Code Here
177
178
179
180
181
182
183
184
185
186
187
      }
      RetrievalEnvironment.topK = kVal;

      // Initialize retrieval environment variables.
      CascadeQueryRunner runner = null;
      MRFBuilder builder = null;
      MRFExpander expander = null;

      try {
        // Get the MRF builder.
        builder = MRFBuilder.get(env, modelNode.cloneNode(true));
View Full Code Here
196
197
198
199
200
201
202
203
204
205
206
    }

    try {
      Node modelNode = d.getElementsByTagName("model").item(0);

      MRFBuilder builder = MRFBuilder.get(mEnv, modelNode.cloneNode(true));

     
      // Set the default number of hits to 2000 because that's what we had in our
      // official TREC 2009 web track runs; otherwise, IF merging approach
      // will give slightly different results, so we won't be able to
View Full Code Here
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
    // Find the best concepts for each of the expansion models.
    for (int modelNum = 0; modelNum < builders.size(); modelNum++) {
      // Get information about this expansion model.
      int curGramSize = gramSizes.get(modelNum);
      MRFBuilder curBuilder = builders.get(modelNum);
      int curFbDocs = fbDocList.get(modelNum);
      int curFbTerms = fbTermList.get(modelNum);

      // Gather Accumulators we're actually going to use for feedback purposes.
      Accumulator[] fbResults = new Accumulator[Math.min(results.length, curFbDocs)];
      for (int i = 0; i < Math.min(results.length, curFbDocs); i++) {
        fbResults[i] = results[i];
      }

      // Sort the Accumulators by docid.
      Arrays.sort(fbResults, new Accumulator.DocnoComparator());

      // Get docnos that correspond to the accumulators.
      int[] docSet = Accumulator.accumulatorsToDocnos(fbResults);

      // Get document vectors for results.
      IntDocVector[] docVecs = env.documentVectors(docSet);

      // Extract vocabulary from results.
      VocabFrequencyPair[] vocab = null;
      try {
        vocab = getVocabulary(docVecs, curGramSize);
      } catch (IOException e) {
        throw new RuntimeException("Error: Unable to fetch the vocabulary!");
      }

      // Priority queue for the concepts associated with this builder.
      PriorityQueue<Accumulator> sortedConcepts = new PriorityQueue<Accumulator>();

      // Score each concept.
      for (int conceptID = 0; conceptID < vocab.length; conceptID++) {
        if (maxCandidates > 0 && conceptID >= maxCandidates) {
          break;
        }

        // The current concept.
        String concept = vocab[conceptID].getKey();

        String[] concepts = concept.split(" ");
        MarkovRandomField conceptMRF = curBuilder.buildMRF(concepts);

        MRFDocumentRanker ranker = new MRFDocumentRanker(conceptMRF, docSet, docSet.length);
        Accumulator[] conceptResults = ranker.rank();
        Arrays.sort(conceptResults, new Accumulator.DocnoComparator());

        float score = 0.0f;
        for (int i = 0; i < conceptResults.length; i++) {
          if (fbResults[i].docno != conceptResults[i].docno) {
            throw new RetrievalException("Error: Mismatch occured in getExpandedMRF!");
          }
          score += Math.exp(fbResults[i].score + conceptResults[i].score);
        }

        int size = sortedConcepts.size();
        if (size < curFbTerms || sortedConcepts.peek().score < score) {
          if (size == curFbTerms) {
            sortedConcepts.poll(); // Remove worst concept.
          }
          sortedConcepts.add(new Accumulator(conceptID, score));
        }
      }

      // Compute the weights of the expanded terms.
      int numTerms = Math.min(curFbTerms, sortedConcepts.size());
      float totalWt = 0.0f;
      Accumulator[] bestConcepts = new Accumulator[numTerms];
      for (int i = 0; i < numTerms; i++) {
        Accumulator a = sortedConcepts.poll();
        bestConcepts[i] = a;
        totalWt += a.score;
      }

      // Add cliques corresponding to best expansion concepts.
      for (int i = 0; i < numTerms; i++) {
        Accumulator a = bestConcepts[i];

        // Construct the MRF corresponding to this concept.
        String[] concepts = vocab[a.docno].getKey().split(" ");
        MarkovRandomField conceptMRF = curBuilder.buildMRF(concepts);

        // Normalized score.
        float normalizedScore = a.score / totalWt;

        // Add cliques.
View Full Code Here
120
121
122
123
124
125
126
127
128
129
130
      Node modelNode = models.get(modelID);
      Node expanderNode = expanders.get(modelID);

      // Initialize retrieval environment variables.
      QueryRunner runner = null;
      MRFBuilder builder = null;
      MRFExpander expander = null;
      try {
        // Get the MRF builder.
        builder = MRFBuilder.get(env, modelNode.cloneNode(true));
View Full Code Here
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
     // compute features for each model
     for(String modelName : modelNames) {
       // build mrf from model node
       Node modelNode = runner.getModel(modelName);
       MRFBuilder builder = MRFBuilder.get(env, modelNode);
       MarkovRandomField mrf = builder.buildMRF(queryText.split("\\s+"));

       // get mrf cliques
      List<Clique> cliques = mrf.getCliques();

      // add parameter name to feature name set
      for(Clique c : cliques) {
        // parameter id
        String paramId = c.getParameter().getName();

        // handle linear importance model weights
        if(importanceModels.size() != 0) {
          for(LinearImportanceModel model : linearImportanceModels) {
            List<MetaFeature> metaFeatures = model.getMetaFeatures();

            for(MetaFeature metaFeat : metaFeatures) {
              // feature id = modelName-metaFeatId-paramId
              String featId = modelName + "-" + metaFeat.getName() + "-" + paramId;
              featureNames.add(featId);
            }
          }
        }

        // feature id = modelName-paramId
        String featId = modelName + "-" + paramId;

        featureNames.add(featId);
      }
     }
   }

   // add judgment feature name
   featureNames.add(JUDGMENT_FEATURE_NAME);

   // print feature name header
   System.out.print(QUERY_FEATURE_NAME + "\t" + DOC_FEATURE_NAME);
   for(String featureName : featureNames) {
     System.out.print("\t" + featureName);
   }
   System.out.println();

   // extract features query-by-query
   for(Entry<String, String> queryEntry : queries.entrySet()) {
     // feature map (docname -> feature name -> feature value)
     SortedMap<String,SortedMap<String,Double>> featureValues = new TreeMap<String,SortedMap<String,Double>>();

     // query id and text
     String qid = queryEntry.getKey();
     String queryText = queryEntry.getValue();

     // compute features for each model
     for(String modelName : modelNames) {
       // build mrf from model node
       Node modelNode = runner.getModel(modelName);
       MRFBuilder builder = MRFBuilder.get(env, modelNode);
       MarkovRandomField mrf = builder.buildMRF(queryText.split("\\s+"));

       // initialize mrf
       mrf.initialize();

       // get mrf cliques
View Full Code Here
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
     // compute features for each model
     for(String modelName : modelNames) {
       // build mrf from model node
       Node modelNode = runner.getModel(modelName);
       MRFBuilder builder = MRFBuilder.get(env, modelNode);
       MarkovRandomField mrf = builder.buildMRF(queryText.split("\\s+"));

       // get mrf cliques
      List<Clique> cliques = mrf.getCliques();

      // add parameter name to feature name set
      for(Clique c : cliques) {
        // parameter id
        String paramId = c.getParameter().getName();

        // handle linear importance model weights
        if(importanceModels.size() != 0) {
          for(LinearImportanceModel model : linearImportanceModels) {
            List<MetaFeature> metaFeatures = model.getMetaFeatures();

            for(MetaFeature metaFeat : metaFeatures) {
              // feature id = modelName-metaFeatId-paramId
              String featId = modelName + "-" + metaFeat.getName() + "-" + paramId;
              featureNames.add(featId);
            }
          }
        }

        // feature id = modelName-paramId
        String featId = modelName + "-" + paramId;

        featureNames.add(featId);
      }
     }
   }

   // add judgment feature name
   featureNames.add(JUDGMENT_FEATURE_NAME);

   // print feature name header
   System.out.print(QUERY_FEATURE_NAME + "\t" + DOC_FEATURE_NAME);
   for(String featureName : featureNames) {
     System.out.print("\t" + featureName);
   }
   System.out.println();

   // extract features query-by-query
   for(Entry<String, String> queryEntry : queries.entrySet()) {
     // feature map (docname -> feature name -> feature value)
     SortedMap<String,SortedMap<String,Operator>> featureValues = new TreeMap<String,SortedMap<String,Operator>>();

     // query id and text
     String qid = queryEntry.getKey();
     String queryText = queryEntry.getValue();

     // compute features for each model
     for(String modelName : modelNames) {
       // build mrf from model node
       Node modelNode = runner.getModel(modelName);
       MRFBuilder builder = MRFBuilder.get(env, modelNode);
       MarkovRandomField mrf = builder.buildMRF(queryText.split("\\s+"));

       // initialize mrf
       mrf.initialize();

       // get mrf cliques
View Full Code Here
120
121
122
123
124
125
126
127
128
129
130
      Node modelNode = models.get(modelID);
      Node expanderNode = expanders.get(modelID);

      // Initialize retrieval environment variables.
      QueryRunner runner = null;
      MRFBuilder builder = null;
      MRFExpander expander = null;
      try {
        // Get the MRF builder.
        builder = MRFBuilder.get(env, modelNode.cloneNode(true));
View Full Code Here
TOP

Related Classes of ivory.smrf.model.builder.MRFBuilder

  • ivory.cascade.model.builder.CascadeFeatureBasedMRFBuilder
  • ivory.cascade.retrieval.CascadeBatchQueryRunner
  • ivory.ltr.ExtractFeatures
  • ivory.server.RetrievalServer
  • ivory.smrf.model.constrained.GreedyConstrainedMRFBuilder
  • ivory.smrf.model.expander.NGramLatentConceptExpander
  • ivory.smrf.retrieval.BatchQueryRunner
  • ivory.smrf.retrieval.distributed.RetrievalServer
  • ivory.core.ConfigurationException
  • ivory.core.exception.ConfigurationException
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.