package
twitter;
import
com.google.common.collect.Lists;
import
com.twitter.hbc.ClientBuilder;
import
com.twitter.hbc.core.Client;
import
com.twitter.hbc.core.Constants;
import
com.twitter.hbc.core.Hosts;
import
com.twitter.hbc.core.HttpHosts;
import
com.twitter.hbc.core.endpoint.StatusesFilterEndpoint;
import
com.twitter.hbc.core.processor.StringDelimitedProcessor;
import
com.twitter.hbc.httpclient.auth.Authentication;
import
com.twitter.hbc.httpclient.auth.OAuth1;
import
org.apache.kafka.clients.producer.KafkaProducer;
import
org.apache.kafka.clients.producer.ProducerConfig;
import
org.apache.kafka.clients.producer.ProducerRecord;
import
org.apache.kafka.common.serialization.StringSerializer;
import
java.util.List;
import
java.util.Properties;
import
java.util.concurrent.BlockingQueue;
import
java.util.concurrent.LinkedBlockingQueue;
import
java.util.concurrent.TimeUnit;
public
class
TwitterProducer {
private
String consumerKey =
"put your consumerKey here"
;
private
String consumerSecret =
"put your consumerSecret here"
;
private
String token =
"put your token here"
;
private
String secret =
"put your secret here"
;
List<String> topics = Lists.newArrayList(
"geeksforgeeks"
,
"java"
,
"kafka"
);
public
static
void
main(String[] args) {
new
TwitterProducer().run();
}
public
TwitterProducer() {
}
public
void
run() {
BlockingQueue<String> msgQueue =
new
LinkedBlockingQueue<>(
1000
);
Client twitterClient = createTwitterClient(msgQueue);
twitterClient.connect();
KafkaProducer<String, String> producer = createKafkaProducer();
sendTweetsToKafka(msgQueue, twitterClient, producer);
}
private
static
void
sendTweetsToKafka(BlockingQueue<String> msgQueue, Client twitterClient,
KafkaProducer<String, String> producer) {
while
(!twitterClient.isDone()) {
String msg =
null
;
try
{
msg = msgQueue.poll(
5
, TimeUnit.SECONDS);
}
catch
(InterruptedException e) {
e.printStackTrace();
twitterClient.stop();
}
if
(msg !=
null
) {
producer.send(
new
ProducerRecord<>(
"twitter_tweets"
,
null
, msg), (recordMetadata, e) -> {
if
(e !=
null
) {
System.out.println(e.getMessage());
}
});
}
}
}
public
Client createTwitterClient(BlockingQueue<String> msgQueue) {
Hosts hosebirdHosts =
new
HttpHosts(Constants.STREAM_HOST);
StatusesFilterEndpoint hosebirdEndpoint =
new
StatusesFilterEndpoint();
hosebirdEndpoint.trackTerms(topics);
Authentication hosebirdAuth =
new
OAuth1(consumerKey, consumerSecret, token, secret);
ClientBuilder builder =
new
ClientBuilder()
.name(
"Hosebird-Client-01"
)
.hosts(hosebirdHosts)
.authentication(hosebirdAuth)
.endpoint(hosebirdEndpoint)
.processor(
new
StringDelimitedProcessor(msgQueue));
Client hosebirdClient = builder.build();
return
hosebirdClient;
}
public
KafkaProducer<String, String> createKafkaProducer() {
String bootstrapServer =
"127.0.0.1:9092"
;
Properties properties =
new
Properties();
properties.setProperty(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, bootstrapServer);
properties.setProperty(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, StringSerializer.
class
.getName());
properties.setProperty(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, StringSerializer.
class
.getName());
KafkaProducer<String, String> producer =
new
KafkaProducer<>(properties);
return
producer;
}
}