python每5分钟从kafka中提取数据的例子

Python  /  管理员 发布于 7年前   184

我就废话不多说了，直接上代码吧！

import syssys.path.append("..")from datetime import datetimefrom utils.kafka2file import KafkaDownloaderimport os"""实现取kafka数据，文件按照取数据的间隔命名如每5分钟从kafka取数据写入文件中，文件名为当前时间加5"""TOPIC = "rtz_queue"HOSTS = "ip:9092,ip:9092"GROUP = "2001"def get_end_time(hour,minute,time_step): if (minute+time_step)%60<60:  if (minute+time_step)%60<10:   return str(hour+int((minute+time_step)/60))+":"+"0"+str((minute+time_step)%60)  else:   return str(hour+int((minute+time_step)/60))+":"+str((minute+time_step)%60) else:  passdef kafkawritefile(time_step,time_num): start = datetime.now() downloader = KafkaDownloader(HOSTS, TOPIC, GROUP) i = 1 while(i<=time_num):  end_time = get_end_time(start.hour, start.minute,i*time_step)  end_time_file = end_time.replace(':', '_')  outfile_path = "/data/tmp/" + end_time_file + ".csv"  if os.path.exists(outfile_path):   os.remove(outfile_path)  writefile = open(outfile_path, 'a+', encoding='utf-8')    for msg in downloader.message():   curr_time = datetime.now()   curr_time = str(curr_time)   split_curr_time = curr_time.split(' ')   curr_time_str = split_curr_time[1][0:5]     if curr_time_str >= str(end_time):      break  i += 1if __name__=='__main__': time_step = 15 time_num = 1 kafkawritefile(time_step,time_num)

以上这篇python每5分钟从kafka中提取数据的例子就是小编分享给大家的全部内容了，希望能给大家一个参考，也希望大家多多支持。