Package kafka.common

Examples of kafka.common.KafkaException

93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
  @Override
  public RecordWriter<K, V> getRecordWriter(TaskAttemptContext context) throws IOException, InterruptedException
  {
    Path outputPath = getOutputPath(context);
    if (outputPath == null)
      throw new KafkaException("no kafka output url specified");
    URI uri = URI.create(outputPath.toString());
    Configuration job = context.getConfiguration();

    Properties props = new Properties();
    String topic;

    props.putAll(kafkaConfigMap);                       // inject default configuration
    for (Map.Entry<String, String> m : job) {           // handle any overrides
      if (!m.getKey().startsWith(KAFKA_CONFIG_PREFIX))
        continue;
      if (m.getKey().equals(KAFKA_URL))
        continue;

      String kafkaKeyName = m.getKey().substring(KAFKA_CONFIG_PREFIX.length()+1);
      props.setProperty(kafkaKeyName, m.getValue());    // set Kafka producer property
    }

    // inject Kafka producer props back into jobconf for easier debugging
    for (Map.Entry<Object, Object> m : props.entrySet()) {
      job.set(KAFKA_CONFIG_PREFIX + "." + m.getKey().toString(), m.getValue().toString());
    }

    // KafkaOutputFormat specific parameters
    final int queueBytes = job.getInt(KAFKA_CONFIG_PREFIX + ".queue.bytes", KAFKA_QUEUE_BYTES);

    if (uri.getScheme().equals("kafka")) {
      // using the direct broker list
      // URL: kafka://<kafka host>/<topic>
      // e.g. kafka://kafka-server:9000,kafka-server2:9000/foobar
      String brokerList = uri.getAuthority();
      props.setProperty("metadata.broker.list", brokerList);
      job.set(KAFKA_CONFIG_PREFIX + ".metadata.broker.list", brokerList);

      if (uri.getPath() == null || uri.getPath().length() <= 1)
        throw new KafkaException("no topic specified in kafka uri");

      topic = uri.getPath().substring(1);               // ignore the initial '/' in the path
      job.set(KAFKA_CONFIG_PREFIX + ".topic", topic);
      log.info(String.format("using kafka broker %s (topic %s)", brokerList, topic));
    } else
      throw new KafkaException("missing scheme from kafka uri (must be kafka://)");

    Producer<Object, byte[]> producer = new Producer<Object, byte[]>(new ProducerConfig(props));
    return new KafkaRecordWriter<K, V>(producer, topic, queueBytes);
  }
View Full Code Here
122
123
124
125
126
127
128
129
130
131
132
   * @return props
   */
  @SuppressWarnings("unchecked")
  public static Props of(String... args) {
    if (args.length % 2 != 0)
      throw new KafkaException(
          "Must have an equal number of keys and values.");
    Map<String, String> vals = new HashMap<String, String>(args.length / 2);
    for (int i = 0; i < args.length; i += 2)
      vals.put(args[i], args[i + 1]);
    return new Props(vals);
View Full Code Here
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
           _limit = _props.getInt("kafka.request.limit", -1);
          
           /*get attemp id*/
           String taskId = _job.get("mapred.task.id");
           if (taskId == null) {
               throw new KafkaException("Configuration does not contain the property mapred.task.id");
           }
           String[] parts = taskId.split("_");
           if (    parts.length != 6 || !parts[0].equals("attempt")
                || (!"m".equals(parts[3]) && !"r".equals(parts[3]))) {
                   throw new KafkaException("TaskAttemptId string : " + taskId + " is not properly formed");
           }
          _attemptId = parts[4]+parts[3];
       }catch (Exception e) {
           throw new IOException (e);
       }
View Full Code Here
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
                    e instanceof SocketTimeoutException ||
                    e instanceof IOException ||
                    e instanceof UnresolvedAddressException
                    ) {
                LOG.warn("Network error when fetching messages:", e);
                throw new KafkaException("Network error when fetching messages: "+e.getMessage());
            } else {
                throw new RuntimeException(e);
            }
        }
        if(fetchResponse.hasError()) {
          short code = fetchResponse.errorCode(topic, partition);
//          if(code == ErrorMapping.OffsetOutOfRangeCode() && config.resetOffsetIfOutOfRange) {
//            long startOffset = getOffset(topic, partition, config.startOffsetTime);
//            offset = startOffset;
//          }
          throw new KafkaException("fetch data from kafka topic["+topic+"] partition["+partition+"] error:" + code);
        }else {
          ByteBufferMessageSet msgs =  fetchResponse.messageSet(topic, partition);
          return msgs;
        }
  }
View Full Code Here
TOP

Related Classes of kafka.common.KafkaException

  • com.alibaba.jstorm.kafka.MessageConsumer
  • kafka.bridge.hadoop.KafkaOutputFormat
  • kafka.etl.KafkaETLRecordReader
  • kafka.etl.Props
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.