Package org.springframework.data.hadoop.fs

Examples of org.springframework.data.hadoop.fs.FsShell

82
83
84
85
86
87
88
89
90
91
92
    }
    configuration = null;
  }

  protected void listFiles(Configuration configuration) {
    @SuppressWarnings("resource")
    FsShell shell = new FsShell(configuration);
    for (FileStatus s : shell.ls(true, "/")) {
      log.info("status " + s);
    }
  }
View Full Code Here
90
91
92
93
94
95
96
97
98
99
100
101
      log.info("status " + s);
    }
  }

  protected void catFile(String path) {
    @SuppressWarnings("resource")
    FsShell shell = new FsShell(configuration);
    Collection<String> text = shell.text(path);
    if (text.size() == 1) {
      log.info(text.iterator().next());
    }
  }
View Full Code Here
102
103
104
105
106
107
108
109
110
111
      writer.write(pojo);
    };
    writer.flush();
    writer.close();

    FsShell fsShell = new FsShell(getConfiguration());
    assertTrue(fsShell.ls(true, path + "/test/" + datasetTemplete.getDatasetName(SimplePojo.class) + "/year=1997/month=05/*.avro").size() >= 2);
    assertTrue(fsShell.ls(true, path + "/test/" + datasetTemplete.getDatasetName(SimplePojo.class) + "/year=1997/month=11/*.avro").size() >= 2);
    assertTrue(fsShell.ls(true, path + "/test/" + datasetTemplete.getDatasetName(SimplePojo.class) + "/year=1997/month=12/*.avro").size() >= 2);
  }
View Full Code Here
132
133
134
135
136
137
138
139
140
141
      writer.write(pojo);
    };
    writer.flush();
    writer.close();

    FsShell fsShell = new FsShell(getConfiguration());
    assertEquals(1, fsShell.ls(true, path + "/test/" + datasetTemplete.getDatasetName(SimplePojo.class) + "/year=1997/month=05/*.avro").size());
    assertEquals(1, fsShell.ls(true, path + "/test/" + datasetTemplete.getDatasetName(SimplePojo.class) + "/year=1997/month=11/*.avro").size());
    assertEquals(1, fsShell.ls(true, path + "/test/" + datasetTemplete.getDatasetName(SimplePojo.class) + "/year=1997/month=12/*.avro").size());
  }
 
View Full Code Here
115
116
117
118
119
120
121
122
123
124
125
      if (cfg == null) {
        log.warn(String.format(
            "No Hadoop Configuration detected; not binding FsShell as variable '%s' to script", name));
      }
      else {
        putIfAbsent(args, name, new FsShell(cfg, fs));
      }
    }

    putIfAbsent(args, "cl", ctx.getClassLoader());
    putIfAbsent(args, "ctxRL", ctx);
View Full Code Here
75
76
77
78
79
80
81
82
83
    Map<String, Object> args = new LinkedHashMap<String, Object>();
    FileSystem fs = FileSystem.get(config);
    SimplerFileSystem sfs = new SimplerFileSystem(fs);
   
    args.put("fs", sfs);
    args.put("fsh", new FsShell(config, sfs));

    eval.evaluate(script, args);
  }
View Full Code Here
30
31
32
33
34
35
36
37
38
39
40
41
42
        "/META-INF/spring/pig-context-password-repository.xml", PigAppWithRepository.class);
    log.info("Pig Application Running");
    context.registerShutdownHook(); 

    String outputDir = "/data/password-repo/output";   
    FsShell fsShell = context.getBean(FsShell.class);
    if (fsShell.test(outputDir)) {
      fsShell.rmr(outputDir);
    }
   
    PasswordRepository repo = context.getBean(PigPasswordRepository.class);
    repo.processPasswordFile("/data/passwd/input");
   
View Full Code Here
20
21
22
23
24
25
26
27
28
29
30
  @OnContainerStart
  public void publicVoidNoArgsMethod() {
    log.info("Hello from HelloPojo");
    log.info("About to list from hdfs root content");

    @SuppressWarnings("resource")
    FsShell shell = new FsShell(configuration);
    for (FileStatus s : shell.ls(false, "/")) {
      log.info(s);
    }
  }
View Full Code Here
56
57
58
59
60
61
62
63
    Assert.notNull(xdEnvironment, "xdEnvironment can not be null");
    this.nameNode = xdEnvironment.getNameNode();

    hadoopConfiguration = new Configuration();
    FileSystem.setDefaultUri(hadoopConfiguration, nameNode);
    shell = new FsShell(hadoopConfiguration);
    dataNodePort = xdEnvironment.getDataNodePort();
  }
View Full Code Here
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
    this.rolloverThresholdInBytes = rolloverThresholdInBytes;
  }

  protected void initializeCounterIfNecessary() {
    if (!initialized) {
      FsShell fsShell = new FsShell(getFileSystem().getConf(), getFileSystem());
      try {
        if (!fsShell.test(getBasePath())) {
          fsShell.mkdir(getBasePath());
        }
        int maxCounter = 0;
        boolean foundFile = false;
        Collection<FileStatus> fileStats = fsShell.ls(this.getBasePath());
        for (FileStatus fileStatus : fileStats) {
          String shortName = fileStatus.getPath().getName();
          int counterFromName = getCounterFromName(shortName);
          if (counterFromName != -1) {
            foundFile = true;
View Full Code Here
TOP

Related Classes of org.springframework.data.hadoop.fs.FsShell

  • com.oreilly.springdata.batch.item.AbstractHdfsItemWriter
  • com.oreilly.springdata.batch.item.SimpleAbstractHdfsItemWriter
  • com.oreilly.springdata.hadoop.filepolling.FsShellWritingMessageHandler
  • com.oreilly.springdata.hadoop.ftp.FsShellWritingMessageHandler
  • com.oreilly.springdata.hadoop.streaming.AbstractHdfsWriter
  • hello.container.HelloPojo
  • org.apache.hadoop.fs.ChecksumFileSystem
  • org.apache.hadoop.fs.FileStatus
  • org.apache.hadoop.fs.FileSystem
  • org.apache.hadoop.fs.FSDataInputStream
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.