Examples of cascading.scheme.Scheme

cascading.scheme.Scheme
A Scheme defines what is stored in a {@link Tap} instance by declaring the {@link Tuple}field names, and alternately parsing or rendering the incoming or outgoing {@link Tuple}stream, respectively.
A Scheme defines the type of resource data will be sourced from or sinked to.
The default sourceFields are {@link Fields#UNKNOWN} and the default sinkFields are {@link Fields#ALL}.
Any given sourceFields only label the values in the {@link Tuple}s as they are sourced. It does not necessarily filter the output since a given implementation may choose to collapse values and ignore keys depending on the format.
If the sinkFields are {@link Fields#ALL}, the Cascading planner will attempt to resolve the actual field names and make them available via the {@link cascading.scheme.SinkCall#getOutgoingEntry()} method. Sometimes this maynot be possible (in the case the {@link Tap#openForWrite(cascading.flow.FlowProcess)} method is called from usercode directly (without planner intervention).
If the sinkFields are a valid selector, the {@link #sink(cascading.flow.FlowProcess,SinkCall)} method willonly see the fields expected.
Setting the {@code numSinkParts} value to 1 (one) attempts to ensure the output resource has only one part.In the case of MapReduce, this is only a suggestion for the Map side, on the Reduce side it does this by setting the number of reducers to the given value. This may affect performance, so be cautioned.
Note that setting numSinkParts does not force the planner to insert a final Reduce operation in the job, so numSinkParts may be ignored entirely if the final job is Map only. To force the Flow to have a final Reduce, add a {@link cascading.pipe.GroupBy} to the assembly before sinking.


    private EsFactory factory = new EsFactory();


    @Test
    public void testCreateScheme() throws IOException {
        Scheme scheme = factory.createScheme(new Fields(), new Properties());
        assertThat(scheme, notNullValue());


        try {
            scheme.sourceConfInit(null, null, null);
            fail();
        } catch (UnsupportedOperationException ex) {
        }


        try {
            scheme.sinkConfInit(null, null, null);
            fail();
        } catch (UnsupportedOperationException ex) {
        }


        try {
            scheme.source(null, null);
            fail();
        } catch (UnsupportedOperationException ex) {
        }
        try {
            scheme.sink(null, null);
            fail();
        } catch (UnsupportedOperationException ex) {
        }


    }

View Full Code Here

    @Test
    public void testCreateTap() {
        Fields fl = new Fields();
        Properties props = new Properties();


        Scheme scheme = factory.createScheme(fl, props);
        Tap tap = factory.createTap(scheme, "somePath", SinkMode.KEEP, props);
        assertThat(tap, notNullValue());
        assertThat(tap.getClass().getName(), containsString("HadoopTap"));
    }

View Full Code Here

        return new SolrScheme(schemeFields, solrCoreDir, maxSegments, dataDirPropertyName);
    }
    
    @Override
    protected Tap<?, ?, ?> makeSolrSink(Fields fields, String path) throws Exception {
        Scheme scheme = new SolrScheme(fields, SOLR_CORE_DIR);
        return new Hfs(scheme, path, SinkMode.REPLACE);
    }

View Full Code Here

    Path path = new Path(inputPath + "/" + schemeFileName);
    FileSystem fs = path.getFileSystem(new Configuration());
    try {
      FSDataInputStream file = fs.open(path);
      ObjectInputStream ois = new ObjectInputStream(file);
      Scheme scheme = (Scheme) ois.readObject();
      Fields fields = (Fields) ois.readObject();
      scheme.setSourceFields(fields);
      ois.close();
      file.close();
      return scheme;
    } catch (ClassNotFoundException e) {
      throw new IOException("Could not read PyCascading file header: " + inputPath + "/"

View Full Code Here

  @Test
  public void testTapSourceConfigDef() throws IOException
    {
    getPlatform().copyFromLocal( inputFileNums20 );


    Scheme scheme = getPlatform().getTestConfigDefScheme();


    Tap source = getPlatform().getTap( scheme, inputFileNums20, SinkMode.KEEP );


    // process -> after sink/sourceConfInit are called
    // default -> Wrapper for all cluster side calls

View Full Code Here


    Pipe pipe = new Pipe( "test" );


    pipe = new Each( pipe, new Insert( new Fields( "value" ), "nada" ), Fields.ALL );


    Scheme scheme = getPlatform().getTestConfigDefScheme();


    Tap sink = getPlatform().getTap( scheme, getOutputPath( "tapsinkconfigdef" ), SinkMode.REPLACE );


    // process -> after sink/sourceConfInit are called
    // default -> Wrapper for all cluster side calls

View Full Code Here

    }


  @Test
  public void testScheme()
    {
    Scheme scheme = new Scheme()
    {
    @Override
    public void sourceConfInit( FlowProcess flowProcess, Tap tap, Object conf )
      {
      }


    @Override
    public void sinkConfInit( FlowProcess flowProcess, Tap tap, Object conf )
      {
      }


    @Override
    public boolean source( FlowProcess flowProcess, SourceCall sourceCall ) throws IOException
      {
      return false;
      }


    @Override
    public void sink( FlowProcess flowProcess, SinkCall sinkCall ) throws IOException
      {
      }
    };


    assertEqualsTrace( "cascading.TraceTest.testScheme(TraceTest.java", scheme.getTrace() );
    }

View Full Code Here

    }


  @Override
  public Scheme getScheme()
    {
    Scheme scheme = super.getScheme();


    if( scheme != null )
      return scheme;


    return taps[ 0 ].getScheme(); // they should all be equivalent per verifyTaps

View Full Code Here

    {
    verifyNotSourcesSinks( flowDef.getCheckpoints(), flowDef.getSources(), flowDef.getSinks(), "checkpoint" );


    for( Tap checkpointTap : flowDef.getCheckpoints().values() )
      {
      Scheme scheme = checkpointTap.getScheme();


      if( scheme.getSourceFields().equals( Fields.UNKNOWN ) && scheme.getSinkFields().equals( Fields.ALL ) )
        continue;


      throw new PlannerException( "checkpoint tap scheme must be undeclared, source fields must be UNKNOWN, and sink fields ALL, got: " + scheme.toString() );
      }


    Set<String> names = new HashSet<String>( asList( Pipe.names( flowTails ) ) );


    for( String name : flowDef.getCheckpoints().keySet() )

View Full Code Here

    Map sources = new HashMap();


    sources.put( "lower", sourceLower );
    sources.put( "upper", sourceUpper );


    Scheme leftScheme = testTempReplaced ? new SequenceFile( new Fields( "num", "lower", "num2", "upper" ) ) : new TextLine( new Fields( "offset", "line" ), new Fields( "lower" ) );
    Tap sinkLeft = new Hfs( leftScheme, "/splitmiddle/left", SinkMode.REPLACE );


    Scheme rightScheme = testTempReplaced ? new SequenceFile( new Fields( "lower" ) ) : new TextLine( new Fields( "offset", "line" ), new Fields( "lower" ) );
    Tap sinkRight = new Hfs( rightScheme, "/splitmiddle/right", SinkMode.REPLACE );


    Map sinks = new HashMap();


    sinks.put( "left", sinkLeft );

View Full Code Here

0 1

TOP

Related Classes of cascading.scheme.Scheme

cascading.flow.hadoop.BuildJobsHadoopPlatformTest

cascading.flow.planner.FlowPlanner

cascading.lingual.catalog.provider.ProviderProxy

cascading.lingual.catalog.SchemaCatalogManager

cascading.property.ConfigDefPlatformTest

cascading.tap.local.LocalTapPlatformTest

cascading.tap.MultiSourceTap

cascading.TraceTest

cascading.TrapPlatformTest

com.scaleunlimited.cascading.scheme.hadoop.SolrSchemeHadoopTest

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.