docs/main/reference/pipelines/source/kafka/index.html

3387 lines
72 KiB
HTML
Raw Blame History

This file contains ambiguous Unicode characters

This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.

<!doctype html>
<html lang="zh" class="no-js">
<head>
<meta charset="utf-8">
<meta name="viewport" content="width=device-width,initial-scale=1">
<meta name="description" content="Loggie Documentation">
<link rel="prev" href="../file/">
<link rel="next" href="../franzkafka/">
<link rel="icon" href="../../../../assets/min-logo.png">
<meta name="generator" content="mkdocs-1.5.3, mkdocs-material-9.1.18">
<title>kafka - </title>
<link rel="stylesheet" href="../../../../assets/stylesheets/main.26e3688c.min.css">
<link rel="preconnect" href="https://fonts.gstatic.com" crossorigin>
<link rel="stylesheet" href="https://fonts.googleapis.com/css?family=Roboto:300,300i,400,400i,700,700i%7CRobot+Mono:400,400i,700,700i&display=fallback">
<style>:root{--md-text-font:"Roboto";--md-code-font:"Robot Mono"}</style>
<link rel="stylesheet" href="../../../../stylesheets/extra.css">
<script>__md_scope=new URL("../../../..",location),__md_hash=e=>[...e].reduce((e,_)=>(e<<5)-e+_.charCodeAt(0),0),__md_get=(e,_=localStorage,t=__md_scope)=>JSON.parse(_.getItem(t.pathname+"."+e)),__md_set=(e,_,t=localStorage,a=__md_scope)=>{try{t.setItem(a.pathname+"."+e,JSON.stringify(_))}catch(e){}}</script>
</head>
<body dir="ltr">
<script>var palette=__md_get("__palette");if(palette&&"object"==typeof palette.color)for(var key of Object.keys(palette.color))document.body.setAttribute("data-md-color-"+key,palette.color[key])</script>
<input class="md-toggle" data-md-toggle="drawer" type="checkbox" id="__drawer" autocomplete="off">
<input class="md-toggle" data-md-toggle="search" type="checkbox" id="__search" autocomplete="off">
<label class="md-overlay" for="__drawer"></label>
<div data-md-component="skip">
<a href="#kafka" class="md-skip">
跳转至
</a>
</div>
<div data-md-component="announce">
</div>
<div data-md-color-scheme="default" data-md-component="outdated" hidden>
</div>
<header class="md-header md-header--shadow md-header--lifted" data-md-component="header">
<nav class="md-header__inner md-grid" aria-label="页眉">
<a href="../../../.." title=" " class="md-header__button md-logo" aria-label=" " data-md-component="logo">
<img src="../../../../assets/loggie.svg" alt="logo">
</a>
<label class="md-header__button md-icon" for="__drawer">
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M3 6h18v2H3V6m0 5h18v2H3v-2m0 5h18v2H3v-2Z"/></svg>
</label>
<div class="md-header__title" data-md-component="header-title">
<div class="md-header__ellipsis">
<div class="md-header__topic">
<span class="md-ellipsis">
</span>
</div>
<div class="md-header__topic" data-md-component="header-topic">
<span class="md-ellipsis">
kafka
</span>
</div>
</div>
</div>
<div class="md-header__option">
<div class="md-select">
<button class="md-header__button md-icon" aria-label="选择当前语言">
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="m12.87 15.07-2.54-2.51.03-.03A17.52 17.52 0 0 0 14.07 6H17V4h-7V2H8v2H1v2h11.17C11.5 7.92 10.44 9.75 9 11.35 8.07 10.32 7.3 9.19 6.69 8h-2c.73 1.63 1.73 3.17 2.98 4.56l-5.09 5.02L4 19l5-5 3.11 3.11.76-2.04M18.5 10h-2L12 22h2l1.12-3h4.75L21 22h2l-4.5-12m-2.62 7 1.62-4.33L19.12 17h-3.24Z"/></svg>
</button>
<div class="md-select__inner">
<ul class="md-select__list">
<li class="md-select__item">
<a href="https://loggie-io.github.io/docs-en/" hreflang="en" class="md-select__link">
English
</a>
</li>
</ul>
</div>
</div>
</div>
<label class="md-header__button md-icon" for="__search">
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M9.5 3A6.5 6.5 0 0 1 16 9.5c0 1.61-.59 3.09-1.56 4.23l.27.27h.79l5 5-1.5 1.5-5-5v-.79l-.27-.27A6.516 6.516 0 0 1 9.5 16 6.5 6.5 0 0 1 3 9.5 6.5 6.5 0 0 1 9.5 3m0 2C7 5 5 7 5 9.5S7 14 9.5 14 14 12 14 9.5 12 5 9.5 5Z"/></svg>
</label>
<div class="md-search" data-md-component="search" role="dialog">
<label class="md-search__overlay" for="__search"></label>
<div class="md-search__inner" role="search">
<form class="md-search__form" name="search">
<input type="text" class="md-search__input" name="query" aria-label="搜索" placeholder="搜索" autocapitalize="off" autocorrect="off" autocomplete="off" spellcheck="false" data-md-component="search-query" required>
<label class="md-search__icon md-icon" for="__search">
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M9.5 3A6.5 6.5 0 0 1 16 9.5c0 1.61-.59 3.09-1.56 4.23l.27.27h.79l5 5-1.5 1.5-5-5v-.79l-.27-.27A6.516 6.516 0 0 1 9.5 16 6.5 6.5 0 0 1 3 9.5 6.5 6.5 0 0 1 9.5 3m0 2C7 5 5 7 5 9.5S7 14 9.5 14 14 12 14 9.5 12 5 9.5 5Z"/></svg>
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M20 11v2H8l5.5 5.5-1.42 1.42L4.16 12l7.92-7.92L13.5 5.5 8 11h12Z"/></svg>
</label>
<nav class="md-search__options" aria-label="查找">
<a href="javascript:void(0)" class="md-search__icon md-icon" title="分享" aria-label="分享" data-clipboard data-clipboard-text="" data-md-component="search-share" tabindex="-1">
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M18 16.08c-.76 0-1.44.3-1.96.77L8.91 12.7c.05-.23.09-.46.09-.7 0-.24-.04-.47-.09-.7l7.05-4.11c.54.5 1.25.81 2.04.81a3 3 0 0 0 3-3 3 3 0 0 0-3-3 3 3 0 0 0-3 3c0 .24.04.47.09.7L8.04 9.81C7.5 9.31 6.79 9 6 9a3 3 0 0 0-3 3 3 3 0 0 0 3 3c.79 0 1.5-.31 2.04-.81l7.12 4.15c-.05.21-.08.43-.08.66 0 1.61 1.31 2.91 2.92 2.91 1.61 0 2.92-1.3 2.92-2.91A2.92 2.92 0 0 0 18 16.08Z"/></svg>
</a>
<button type="reset" class="md-search__icon md-icon" title="清空当前内容" aria-label="清空当前内容" tabindex="-1">
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M19 6.41 17.59 5 12 10.59 6.41 5 5 6.41 10.59 12 5 17.59 6.41 19 12 13.41 17.59 19 19 17.59 13.41 12 19 6.41Z"/></svg>
</button>
</nav>
<div class="md-search__suggest" data-md-component="search-suggest"></div>
</form>
<div class="md-search__output">
<div class="md-search__scrollwrap" data-md-scrollfix>
<div class="md-search-result" data-md-component="search-result">
<div class="md-search-result__meta">
正在初始化搜索引擎
</div>
<ol class="md-search-result__list" role="presentation"></ol>
</div>
</div>
</div>
</div>
</div>
<div class="md-header__source">
<a href="https://github.com/loggie-io/loggie" title="前往仓库" class="md-source" data-md-component="source">
<div class="md-source__icon md-icon">
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 496 512"><!--! Font Awesome Free 6.4.0 by @fontawesome - https://fontawesome.com License - https://fontawesome.com/license/free (Icons: CC BY 4.0, Fonts: SIL OFL 1.1, Code: MIT License) Copyright 2023 Fonticons, Inc.--><path d="M165.9 397.4c0 2-2.3 3.6-5.2 3.6-3.3.3-5.6-1.3-5.6-3.6 0-2 2.3-3.6 5.2-3.6 3-.3 5.6 1.3 5.6 3.6zm-31.1-4.5c-.7 2 1.3 4.3 4.3 4.9 2.6 1 5.6 0 6.2-2s-1.3-4.3-4.3-5.2c-2.6-.7-5.5.3-6.2 2.3zm44.2-1.7c-2.9.7-4.9 2.6-4.6 4.9.3 2 2.9 3.3 5.9 2.6 2.9-.7 4.9-2.6 4.6-4.6-.3-1.9-3-3.2-5.9-2.9zM244.8 8C106.1 8 0 113.3 0 252c0 110.9 69.8 205.8 169.5 239.2 12.8 2.3 17.3-5.6 17.3-12.1 0-6.2-.3-40.4-.3-61.4 0 0-70 15-84.7-29.8 0 0-11.4-29.1-27.8-36.6 0 0-22.9-15.7 1.6-15.4 0 0 24.9 2 38.6 25.8 21.9 38.6 58.6 27.5 72.9 20.9 2.3-16 8.8-27.1 16-33.7-55.9-6.2-112.3-14.3-112.3-110.5 0-27.5 7.6-41.3 23.6-58.9-2.6-6.5-11.1-33.3 2.6-67.9 20.9-6.5 69 27 69 27 20-5.6 41.5-8.5 62.8-8.5s42.8 2.9 62.8 8.5c0 0 48.1-33.6 69-27 13.7 34.7 5.2 61.4 2.6 67.9 16 17.7 25.8 31.5 25.8 58.9 0 96.5-58.9 104.2-114.8 110.5 9.2 7.9 17 22.9 17 46.4 0 33.7-.3 75.4-.3 83.6 0 6.5 4.6 14.4 17.3 12.1C428.2 457.8 496 362.9 496 252 496 113.3 383.5 8 244.8 8zM97.2 352.9c-1.3 1-1 3.3.7 5.2 1.6 1.6 3.9 2.3 5.2 1 1.3-1 1-3.3-.7-5.2-1.6-1.6-3.9-2.3-5.2-1zm-10.8-8.1c-.7 1.3.3 2.9 2.3 3.9 1.6 1 3.6.7 4.3-.7.7-1.3-.3-2.9-2.3-3.9-2-.6-3.6-.3-4.3.7zm32.4 35.6c-1.6 1.3-1 4.3 1.3 6.2 2.3 2.3 5.2 2.6 6.5 1 1.3-1.3.7-4.3-1.3-6.2-2.2-2.3-5.2-2.6-6.5-1zm-11.4-14.7c-1.6 1-1.6 3.6 0 5.9 1.6 2.3 4.3 3.3 5.6 2.3 1.6-1.3 1.6-3.9 0-6.2-1.4-2.3-4-3.3-5.6-2z"/></svg>
</div>
<div class="md-source__repository">
loggie-io/loggie
</div>
</a>
</div>
</nav>
<nav class="md-tabs" aria-label="标签" data-md-component="tabs">
<div class="md-grid">
<ul class="md-tabs__list">
<li class="md-tabs__item">
<a href="../../../.." class="md-tabs__link">
Home
</a>
</li>
<li class="md-tabs__item">
<a href="../../../../getting-started/overview/" class="md-tabs__link">
快速上手
</a>
</li>
<li class="md-tabs__item">
<a href="../../../../user-guide/" class="md-tabs__link">
用户指南
</a>
</li>
<li class="md-tabs__item">
<a href="../../../" class="md-tabs__link md-tabs__link--active">
组件配置
</a>
</li>
<li class="md-tabs__item">
<a href="../../../../developer-guide/contributing/" class="md-tabs__link">
开发手册
</a>
</li>
<li class="md-tabs__item">
<a href="../../../../blog/" class="md-tabs__link">
Blog
</a>
</li>
</ul>
</div>
</nav>
</header>
<div class="md-container" data-md-component="container">
<main class="md-main" data-md-component="main">
<div class="md-main__inner md-grid">
<div class="md-sidebar md-sidebar--primary" data-md-component="sidebar" data-md-type="navigation" >
<div class="md-sidebar__scrollwrap">
<div class="md-sidebar__inner">
<nav class="md-nav md-nav--primary md-nav--lifted" aria-label="导航栏" data-md-level="0">
<label class="md-nav__title" for="__drawer">
<a href="../../../.." title=" " class="md-nav__button md-logo" aria-label=" " data-md-component="logo">
<img src="../../../../assets/loggie.svg" alt="logo">
</a>
</label>
<div class="md-nav__source">
<a href="https://github.com/loggie-io/loggie" title="前往仓库" class="md-source" data-md-component="source">
<div class="md-source__icon md-icon">
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 496 512"><!--! Font Awesome Free 6.4.0 by @fontawesome - https://fontawesome.com License - https://fontawesome.com/license/free (Icons: CC BY 4.0, Fonts: SIL OFL 1.1, Code: MIT License) Copyright 2023 Fonticons, Inc.--><path d="M165.9 397.4c0 2-2.3 3.6-5.2 3.6-3.3.3-5.6-1.3-5.6-3.6 0-2 2.3-3.6 5.2-3.6 3-.3 5.6 1.3 5.6 3.6zm-31.1-4.5c-.7 2 1.3 4.3 4.3 4.9 2.6 1 5.6 0 6.2-2s-1.3-4.3-4.3-5.2c-2.6-.7-5.5.3-6.2 2.3zm44.2-1.7c-2.9.7-4.9 2.6-4.6 4.9.3 2 2.9 3.3 5.9 2.6 2.9-.7 4.9-2.6 4.6-4.6-.3-1.9-3-3.2-5.9-2.9zM244.8 8C106.1 8 0 113.3 0 252c0 110.9 69.8 205.8 169.5 239.2 12.8 2.3 17.3-5.6 17.3-12.1 0-6.2-.3-40.4-.3-61.4 0 0-70 15-84.7-29.8 0 0-11.4-29.1-27.8-36.6 0 0-22.9-15.7 1.6-15.4 0 0 24.9 2 38.6 25.8 21.9 38.6 58.6 27.5 72.9 20.9 2.3-16 8.8-27.1 16-33.7-55.9-6.2-112.3-14.3-112.3-110.5 0-27.5 7.6-41.3 23.6-58.9-2.6-6.5-11.1-33.3 2.6-67.9 20.9-6.5 69 27 69 27 20-5.6 41.5-8.5 62.8-8.5s42.8 2.9 62.8 8.5c0 0 48.1-33.6 69-27 13.7 34.7 5.2 61.4 2.6 67.9 16 17.7 25.8 31.5 25.8 58.9 0 96.5-58.9 104.2-114.8 110.5 9.2 7.9 17 22.9 17 46.4 0 33.7-.3 75.4-.3 83.6 0 6.5 4.6 14.4 17.3 12.1C428.2 457.8 496 362.9 496 252 496 113.3 383.5 8 244.8 8zM97.2 352.9c-1.3 1-1 3.3.7 5.2 1.6 1.6 3.9 2.3 5.2 1 1.3-1 1-3.3-.7-5.2-1.6-1.6-3.9-2.3-5.2-1zm-10.8-8.1c-.7 1.3.3 2.9 2.3 3.9 1.6 1 3.6.7 4.3-.7.7-1.3-.3-2.9-2.3-3.9-2-.6-3.6-.3-4.3.7zm32.4 35.6c-1.6 1.3-1 4.3 1.3 6.2 2.3 2.3 5.2 2.6 6.5 1 1.3-1.3.7-4.3-1.3-6.2-2.2-2.3-5.2-2.6-6.5-1zm-11.4-14.7c-1.6 1-1.6 3.6 0 5.9 1.6 2.3 4.3 3.3 5.6 2.3 1.6-1.3 1.6-3.9 0-6.2-1.4-2.3-4-3.3-5.6-2z"/></svg>
</div>
<div class="md-source__repository">
loggie-io/loggie
</div>
</a>
</div>
<ul class="md-nav__list" data-md-scrollfix>
<li class="md-nav__item">
<a href="../../../.." class="md-nav__link">
Home
</a>
</li>
<li class="md-nav__item md-nav__item--nested">
<input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_2" >
<label class="md-nav__link" for="__nav_2" id="__nav_2_label" tabindex="0">
快速上手
<span class="md-nav__icon md-icon"></span>
</label>
<nav class="md-nav" data-md-level="1" aria-labelledby="__nav_2_label" aria-expanded="false">
<label class="md-nav__title" for="__nav_2">
<span class="md-nav__icon md-icon"></span>
快速上手
</label>
<ul class="md-nav__list" data-md-scrollfix>
<li class="md-nav__item">
<a href="../../../../getting-started/overview/" class="md-nav__link">
Overview
</a>
</li>
<li class="md-nav__item">
<a href="../../../../getting-started/intro/core-concept/" class="md-nav__link">
核心概念
</a>
</li>
<li class="md-nav__item md-nav__item--section md-nav__item--nested">
<input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_2_3" >
<label class="md-nav__link" for="__nav_2_3" id="__nav_2_3_label" tabindex="0">
快速上手
<span class="md-nav__icon md-icon"></span>
</label>
<nav class="md-nav" data-md-level="2" aria-labelledby="__nav_2_3_label" aria-expanded="false">
<label class="md-nav__title" for="__nav_2_3">
<span class="md-nav__icon md-icon"></span>
快速上手
</label>
<ul class="md-nav__list" data-md-scrollfix>
<li class="md-nav__item">
<a href="../../../../getting-started/quick-start/quick-start/" class="md-nav__link">
Overview
</a>
</li>
<li class="md-nav__item">
<a href="../../../../getting-started/quick-start/kubernetes/" class="md-nav__link">
Kubernetes环境
</a>
</li>
<li class="md-nav__item">
<a href="../../../../getting-started/quick-start/node/" class="md-nav__link">
主机环境
</a>
</li>
</ul>
</nav>
</li>
<li class="md-nav__item md-nav__item--section md-nav__item--nested">
<input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_2_4" >
<label class="md-nav__link" for="__nav_2_4" id="__nav_2_4_label" tabindex="0">
部署
<span class="md-nav__icon md-icon"></span>
</label>
<nav class="md-nav" data-md-level="2" aria-labelledby="__nav_2_4_label" aria-expanded="false">
<label class="md-nav__title" for="__nav_2_4">
<span class="md-nav__icon md-icon"></span>
部署
</label>
<ul class="md-nav__list" data-md-scrollfix>
<li class="md-nav__item">
<a href="../../../../getting-started/install/kubernetes/" class="md-nav__link">
Kubernetes部署
</a>
</li>
<li class="md-nav__item">
<a href="../../../../getting-started/install/node/" class="md-nav__link">
主机部署
</a>
</li>
</ul>
</nav>
</li>
<li class="md-nav__item md-nav__item--section md-nav__item--nested">
<input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_2_5" >
<label class="md-nav__link" for="__nav_2_5" id="__nav_2_5_label" tabindex="0">
RoadMap
<span class="md-nav__icon md-icon"></span>
</label>
<nav class="md-nav" data-md-level="2" aria-labelledby="__nav_2_5_label" aria-expanded="false">
<label class="md-nav__title" for="__nav_2_5">
<span class="md-nav__icon md-icon"></span>
RoadMap
</label>
<ul class="md-nav__list" data-md-scrollfix>
<li class="md-nav__item">
<a href="../../../../getting-started/roadmap/roadmap-2023/" class="md-nav__link">
2023
</a>
</li>
</ul>
</nav>
</li>
</ul>
</nav>
</li>
<li class="md-nav__item md-nav__item--nested">
<input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_3" >
<label class="md-nav__link" for="__nav_3" id="__nav_3_label" tabindex="0">
用户指南
<span class="md-nav__icon md-icon"></span>
</label>
<nav class="md-nav" data-md-level="1" aria-labelledby="__nav_3_label" aria-expanded="false">
<label class="md-nav__title" for="__nav_3">
<span class="md-nav__icon md-icon"></span>
用户指南
</label>
<ul class="md-nav__list" data-md-scrollfix>
<li class="md-nav__item">
<a href="../../../../user-guide/" class="md-nav__link">
Overview
</a>
</li>
<li class="md-nav__item md-nav__item--section md-nav__item--nested">
<input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_3_2" >
<label class="md-nav__link" for="__nav_3_2" id="__nav_3_2_label" tabindex="0">
架构与特性
<span class="md-nav__icon md-icon"></span>
</label>
<nav class="md-nav" data-md-level="2" aria-labelledby="__nav_3_2_label" aria-expanded="false">
<label class="md-nav__title" for="__nav_3_2">
<span class="md-nav__icon md-icon"></span>
架构与特性
</label>
<ul class="md-nav__list" data-md-scrollfix>
<li class="md-nav__item">
<a href="../../../../user-guide/architecture/background/" class="md-nav__link">
诞生背景
</a>
</li>
<li class="md-nav__item">
<a href="../../../../user-guide/architecture/core-arch/" class="md-nav__link">
设计架构
</a>
</li>
<li class="md-nav__item">
<a href="../../../../user-guide/architecture/schema/" class="md-nav__link">
数据格式
</a>
</li>
<li class="md-nav__item">
<a href="../../../../user-guide/architecture/advantages/" class="md-nav__link">
优势与特性
</a>
</li>
<li class="md-nav__item">
<a href="../../../../user-guide/architecture/compare/" class="md-nav__link">
开源项目对比
</a>
</li>
</ul>
</nav>
</li>
<li class="md-nav__item md-nav__item--section md-nav__item--nested">
<input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_3_3" >
<label class="md-nav__link" for="__nav_3_3" id="__nav_3_3_label" tabindex="0">
Kubernetes
<span class="md-nav__icon md-icon"></span>
</label>
<nav class="md-nav" data-md-level="2" aria-labelledby="__nav_3_3_label" aria-expanded="false">
<label class="md-nav__title" for="__nav_3_3">
<span class="md-nav__icon md-icon"></span>
Kubernetes
</label>
<ul class="md-nav__list" data-md-scrollfix>
<li class="md-nav__item">
<a href="../../../../user-guide/use-in-kubernetes/general-usage/" class="md-nav__link">
Kubernetes下的日志采集
</a>
</li>
<li class="md-nav__item">
<a href="../../../../user-guide/use-in-kubernetes/collect-container-logs/" class="md-nav__link">
Loggie采集容器日志
</a>
</li>
<li class="md-nav__item">
<a href="../../../../user-guide/use-in-kubernetes/collect-node-logs/" class="md-nav__link">
Loggie采集Node日志
</a>
</li>
<li class="md-nav__item">
<a href="../../../../user-guide/use-in-kubernetes/sidecar/" class="md-nav__link">
Sidecar方式采集日志
</a>
</li>
<li class="md-nav__item">
<a href="../../../../user-guide/use-in-kubernetes/kube-event-source/" class="md-nav__link">
采集Kubernetes Events
</a>
</li>
</ul>
</nav>
</li>
<li class="md-nav__item md-nav__item--section md-nav__item--nested">
<input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_3_4" >
<label class="md-nav__link" for="__nav_3_4" id="__nav_3_4_label" tabindex="0">
最佳实践
<span class="md-nav__icon md-icon"></span>
</label>
<nav class="md-nav" data-md-level="2" aria-labelledby="__nav_3_4_label" aria-expanded="false">
<label class="md-nav__title" for="__nav_3_4">
<span class="md-nav__icon md-icon"></span>
最佳实践
</label>
<ul class="md-nav__list" data-md-scrollfix>
<li class="md-nav__item">
<a href="../../../../user-guide/best-practice/log-collect-rotate/" class="md-nav__link">
日志rotate和日志采集
</a>
</li>
<li class="md-nav__item">
<a href="../../../../user-guide/best-practice/log-enrich/" class="md-nav__link">
日志格式与元信息字段
</a>
</li>
<li class="md-nav__item">
<a href="../../../../user-guide/best-practice/log-process/" class="md-nav__link">
日志切分处理
</a>
</li>
<li class="md-nav__item">
<a href="../../../../user-guide/best-practice/aggregator/" class="md-nav__link">
使用Loggie中转机
</a>
</li>
<li class="md-nav__item">
<a href="../../../../user-guide/best-practice/concurrency/" class="md-nav__link">
自适应sink流量控制
</a>
</li>
</ul>
</nav>
</li>
<li class="md-nav__item md-nav__item--section md-nav__item--nested">
<input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_3_5" >
<label class="md-nav__link" for="__nav_3_5" id="__nav_3_5_label" tabindex="0">
监控报警
<span class="md-nav__icon md-icon"></span>
</label>
<nav class="md-nav" data-md-level="2" aria-labelledby="__nav_3_5_label" aria-expanded="false">
<label class="md-nav__title" for="__nav_3_5">
<span class="md-nav__icon md-icon"></span>
监控报警
</label>
<ul class="md-nav__list" data-md-scrollfix>
<li class="md-nav__item">
<a href="../../../../user-guide/monitor/loggie-monitor/" class="md-nav__link">
Loggie的监控与报警
</a>
</li>
<li class="md-nav__item">
<a href="../../../../user-guide/monitor/service-log-alarm/" class="md-nav__link">
日志报警
</a>
</li>
</ul>
</nav>
</li>
<li class="md-nav__item md-nav__item--section md-nav__item--nested">
<input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_3_6" >
<label class="md-nav__link" for="__nav_3_6" id="__nav_3_6_label" tabindex="0">
企业实战
<span class="md-nav__icon md-icon"></span>
</label>
<nav class="md-nav" data-md-level="2" aria-labelledby="__nav_3_6_label" aria-expanded="false">
<label class="md-nav__title" for="__nav_3_6">
<span class="md-nav__icon md-icon"></span>
企业实战
</label>
<ul class="md-nav__list" data-md-scrollfix>
<li class="md-nav__item">
<a href="../../../../user-guide/enterprise-practice/architecture-and-evolution/" class="md-nav__link">
日志系统架构与演进
</a>
</li>
<li class="md-nav__item">
<a href="../../../../user-guide/enterprise-practice/sls/" class="md-nav__link">
使用阿里云可观测统一存储SLS
</a>
</li>
</ul>
</nav>
</li>
<li class="md-nav__item md-nav__item--section md-nav__item--nested">
<input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_3_7" >
<label class="md-nav__link" for="__nav_3_7" id="__nav_3_7_label" tabindex="0">
运维排障
<span class="md-nav__icon md-icon"></span>
</label>
<nav class="md-nav" data-md-level="2" aria-labelledby="__nav_3_7_label" aria-expanded="false">
<label class="md-nav__title" for="__nav_3_7">
<span class="md-nav__icon md-icon"></span>
运维排障
</label>
<ul class="md-nav__list" data-md-scrollfix>
<li class="md-nav__item">
<a href="../../../../user-guide/troubleshot/log-collection/" class="md-nav__link">
日志采集快速排障指南
</a>
</li>
<li class="md-nav__item">
<a href="../../../../user-guide/troubleshot/problems/" class="md-nav__link">
问题案例
</a>
</li>
</ul>
</nav>
</li>
</ul>
</nav>
</li>
<li class="md-nav__item md-nav__item--active md-nav__item--nested">
<input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_4" checked>
<label class="md-nav__link" for="__nav_4" id="__nav_4_label" tabindex="0">
组件配置
<span class="md-nav__icon md-icon"></span>
</label>
<nav class="md-nav" data-md-level="1" aria-labelledby="__nav_4_label" aria-expanded="true">
<label class="md-nav__title" for="__nav_4">
<span class="md-nav__icon md-icon"></span>
组件配置
</label>
<ul class="md-nav__list" data-md-scrollfix>
<li class="md-nav__item">
<a href="../../../" class="md-nav__link">
Overview
</a>
</li>
<li class="md-nav__item">
<a href="../../../global/args/" class="md-nav__link">
启动参数
</a>
</li>
<li class="md-nav__item">
<a href="../../../global/subcmd/" class="md-nav__link">
子命令
</a>
</li>
<li class="md-nav__item">
<a href="../../../global/var/" class="md-nav__link">
字段变量
</a>
</li>
<li class="md-nav__item md-nav__item--section md-nav__item--nested">
<input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_4_5" >
<label class="md-nav__link" for="__nav_4_5" id="__nav_4_5_label" tabindex="0">
系统配置
<span class="md-nav__icon md-icon"></span>
</label>
<nav class="md-nav" data-md-level="2" aria-labelledby="__nav_4_5_label" aria-expanded="false">
<label class="md-nav__title" for="__nav_4_5">
<span class="md-nav__icon md-icon"></span>
系统配置
</label>
<ul class="md-nav__list" data-md-scrollfix>
<li class="md-nav__item">
<a href="../../../global/monitor/" class="md-nav__link">
monitor
</a>
</li>
<li class="md-nav__item">
<a href="../../../global/discovery/" class="md-nav__link">
discovery
</a>
</li>
<li class="md-nav__item">
<a href="../../../global/reload/" class="md-nav__link">
reload
</a>
</li>
<li class="md-nav__item">
<a href="../../../global/defaults/" class="md-nav__link">
defaults
</a>
</li>
<li class="md-nav__item">
<a href="../../../global/http/" class="md-nav__link">
http
</a>
</li>
<li class="md-nav__item">
<a href="../../../global/db/" class="md-nav__link">
db
</a>
</li>
</ul>
</nav>
</li>
<li class="md-nav__item md-nav__item--section md-nav__item--nested">
<input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_4_6" >
<label class="md-nav__link" for="__nav_4_6" id="__nav_4_6_label" tabindex="0">
Kubernetes CRD
<span class="md-nav__icon md-icon"></span>
</label>
<nav class="md-nav" data-md-level="2" aria-labelledby="__nav_4_6_label" aria-expanded="false">
<label class="md-nav__title" for="__nav_4_6">
<span class="md-nav__icon md-icon"></span>
Kubernetes CRD
</label>
<ul class="md-nav__list" data-md-scrollfix>
<li class="md-nav__item">
<a href="../../../discovery/kubernetes/logconfig/" class="md-nav__link">
LogConfig
</a>
</li>
<li class="md-nav__item">
<a href="../../../discovery/kubernetes/clusterlogconfig/" class="md-nav__link">
ClusterLogConfig
</a>
</li>
<li class="md-nav__item">
<a href="../../../discovery/kubernetes/sink/" class="md-nav__link">
Sink
</a>
</li>
<li class="md-nav__item">
<a href="../../../discovery/kubernetes/interceptors/" class="md-nav__link">
Interceptor
</a>
</li>
</ul>
</nav>
</li>
<li class="md-nav__item md-nav__item--active md-nav__item--section md-nav__item--nested">
<input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_4_7" checked>
<label class="md-nav__link" for="__nav_4_7" id="__nav_4_7_label" tabindex="0">
Source
<span class="md-nav__icon md-icon"></span>
</label>
<nav class="md-nav" data-md-level="2" aria-labelledby="__nav_4_7_label" aria-expanded="true">
<label class="md-nav__title" for="__nav_4_7">
<span class="md-nav__icon md-icon"></span>
Source
</label>
<ul class="md-nav__list" data-md-scrollfix>
<li class="md-nav__item">
<a href="../overview/" class="md-nav__link">
Overview
</a>
</li>
<li class="md-nav__item">
<a href="../file/" class="md-nav__link">
file
</a>
</li>
<li class="md-nav__item md-nav__item--active">
<input class="md-nav__toggle md-toggle" type="checkbox" id="__toc">
<label class="md-nav__link md-nav__link--active" for="__toc">
kafka
<span class="md-nav__icon md-icon"></span>
</label>
<a href="./" class="md-nav__link md-nav__link--active">
kafka
</a>
<nav class="md-nav md-nav--secondary" aria-label="目录">
<label class="md-nav__title" for="__toc">
<span class="md-nav__icon md-icon"></span>
目录
</label>
<ul class="md-nav__list" data-md-component="toc" data-md-scrollfix>
<li class="md-nav__item">
<a href="#brokers" class="md-nav__link">
brokers
</a>
</li>
<li class="md-nav__item">
<a href="#topic" class="md-nav__link">
topic
</a>
</li>
<li class="md-nav__item">
<a href="#topics" class="md-nav__link">
topics
</a>
</li>
<li class="md-nav__item">
<a href="#groupid" class="md-nav__link">
groupId
</a>
</li>
<li class="md-nav__item">
<a href="#clientid" class="md-nav__link">
clientId
</a>
</li>
<li class="md-nav__item">
<a href="#worker" class="md-nav__link">
worker
</a>
</li>
<li class="md-nav__item">
<a href="#addonmeta" class="md-nav__link">
addonMeta
</a>
</li>
<li class="md-nav__item">
<a href="#queuecapacity" class="md-nav__link">
queueCapacity
</a>
</li>
<li class="md-nav__item">
<a href="#minacceptedbytes" class="md-nav__link">
minAcceptedBytes
</a>
</li>
<li class="md-nav__item">
<a href="#maxacceptedbytes" class="md-nav__link">
maxAcceptedBytes
</a>
</li>
<li class="md-nav__item">
<a href="#readmaxattempts" class="md-nav__link">
readMaxAttempts
</a>
</li>
<li class="md-nav__item">
<a href="#maxpollwait" class="md-nav__link">
maxPollWait
</a>
</li>
<li class="md-nav__item">
<a href="#readbackoffmin" class="md-nav__link">
readBackoffMin
</a>
</li>
<li class="md-nav__item">
<a href="#readbackoffmax" class="md-nav__link">
readBackoffMax
</a>
</li>
<li class="md-nav__item">
<a href="#enableautocommit" class="md-nav__link">
enableAutoCommit
</a>
</li>
<li class="md-nav__item">
<a href="#autocommitinterval" class="md-nav__link">
autoCommitInterval
</a>
</li>
<li class="md-nav__item">
<a href="#autooffsetreset" class="md-nav__link">
autoOffsetReset
</a>
</li>
<li class="md-nav__item">
<a href="#sasl" class="md-nav__link">
sasl
</a>
</li>
</ul>
</nav>
</li>
<li class="md-nav__item">
<a href="../franzkafka/" class="md-nav__link">
kafka(franz)
</a>
</li>
<li class="md-nav__item">
<a href="../elasticsearch/" class="md-nav__link">
elasticsearch
</a>
</li>
<li class="md-nav__item">
<a href="../kube-event/" class="md-nav__link">
kubeEvent
</a>
</li>
<li class="md-nav__item">
<a href="../grpc/" class="md-nav__link">
grpc
</a>
</li>
<li class="md-nav__item">
<a href="../prometheus-exporter/" class="md-nav__link">
prometheusExporter
</a>
</li>
<li class="md-nav__item">
<a href="../unix/" class="md-nav__link">
unix
</a>
</li>
<li class="md-nav__item">
<a href="../dev/" class="md-nav__link">
dev
</a>
</li>
</ul>
</nav>
</li>
<li class="md-nav__item md-nav__item--section md-nav__item--nested">
<input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_4_8" >
<label class="md-nav__link" for="__nav_4_8" id="__nav_4_8_label" tabindex="0">
Sink
<span class="md-nav__icon md-icon"></span>
</label>
<nav class="md-nav" data-md-level="2" aria-labelledby="__nav_4_8_label" aria-expanded="false">
<label class="md-nav__title" for="__nav_4_8">
<span class="md-nav__icon md-icon"></span>
Sink
</label>
<ul class="md-nav__list" data-md-scrollfix>
<li class="md-nav__item">
<a href="../../sink/overview/" class="md-nav__link">
Overview
</a>
</li>
<li class="md-nav__item">
<a href="../../sink/elasticsearch/" class="md-nav__link">
elasticsearch
</a>
</li>
<li class="md-nav__item">
<a href="../../sink/kafka/" class="md-nav__link">
kafka
</a>
</li>
<li class="md-nav__item">
<a href="../../sink/franzkafka/" class="md-nav__link">
kafka(franz)
</a>
</li>
<li class="md-nav__item">
<a href="../../sink/loki/" class="md-nav__link">
loki
</a>
</li>
<li class="md-nav__item">
<a href="../../sink/pulsar/" class="md-nav__link">
pulsar
</a>
</li>
<li class="md-nav__item">
<a href="../../sink/rocketmq/" class="md-nav__link">
rocketmq
</a>
</li>
<li class="md-nav__item">
<a href="../../sink/grpc/" class="md-nav__link">
grpc
</a>
</li>
<li class="md-nav__item">
<a href="../../sink/file/" class="md-nav__link">
file
</a>
</li>
<li class="md-nav__item">
<a href="../../sink/dev/" class="md-nav__link">
dev
</a>
</li>
<li class="md-nav__item">
<a href="../../sink/sls/" class="md-nav__link">
sls
</a>
</li>
<li class="md-nav__item">
<a href="../../sink/webhook/" class="md-nav__link">
alertwebhook
</a>
</li>
<li class="md-nav__item">
<a href="../../sink/zinc/" class="md-nav__link">
zinc
</a>
</li>
</ul>
</nav>
</li>
<li class="md-nav__item md-nav__item--section md-nav__item--nested">
<input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_4_9" >
<label class="md-nav__link" for="__nav_4_9" id="__nav_4_9_label" tabindex="0">
Interceptor
<span class="md-nav__icon md-icon"></span>
</label>
<nav class="md-nav" data-md-level="2" aria-labelledby="__nav_4_9_label" aria-expanded="false">
<label class="md-nav__title" for="__nav_4_9">
<span class="md-nav__icon md-icon"></span>
Interceptor
</label>
<ul class="md-nav__list" data-md-scrollfix>
<li class="md-nav__item">
<a href="../../interceptor/overview/" class="md-nav__link">
Overview
</a>
</li>
<li class="md-nav__item">
<a href="../../interceptor/schema/" class="md-nav__link">
schema
</a>
</li>
<li class="md-nav__item">
<a href="../../interceptor/transformer/" class="md-nav__link">
transformer
</a>
</li>
<li class="md-nav__item">
<a href="../../interceptor/limit/" class="md-nav__link">
rateLimit
</a>
</li>
<li class="md-nav__item">
<a href="../../interceptor/addhostmeta/" class="md-nav__link">
addHostMeta
</a>
</li>
<li class="md-nav__item">
<a href="../../interceptor/addk8smeta/" class="md-nav__link">
addK8sMeta
</a>
</li>
<li class="md-nav__item">
<a href="../../interceptor/logalert/" class="md-nav__link">
logAlert
</a>
</li>
<li class="md-nav__item">
<a href="../../interceptor/metrics/" class="md-nav__link">
metrics
</a>
</li>
<li class="md-nav__item">
<a href="../../interceptor/retry/" class="md-nav__link">
retry
</a>
</li>
<li class="md-nav__item">
<a href="../../interceptor/maxbytes/" class="md-nav__link">
maxbytes
</a>
</li>
<li class="md-nav__item">
<a href="../../interceptor/normalize/" class="md-nav__link">
normalize
</a>
</li>
</ul>
</nav>
</li>
<li class="md-nav__item md-nav__item--section md-nav__item--nested">
<input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_4_10" >
<label class="md-nav__link" for="__nav_4_10" id="__nav_4_10_label" tabindex="0">
Queue
<span class="md-nav__icon md-icon"></span>
</label>
<nav class="md-nav" data-md-level="2" aria-labelledby="__nav_4_10_label" aria-expanded="false">
<label class="md-nav__title" for="__nav_4_10">
<span class="md-nav__icon md-icon"></span>
Queue
</label>
<ul class="md-nav__list" data-md-scrollfix>
<li class="md-nav__item">
<a href="../../queue/channel/" class="md-nav__link">
channel
</a>
</li>
<li class="md-nav__item">
<a href="../../queue/memory/" class="md-nav__link">
memory
</a>
</li>
</ul>
</nav>
</li>
<li class="md-nav__item md-nav__item--section md-nav__item--nested">
<input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_4_11" >
<label class="md-nav__link" for="__nav_4_11" id="__nav_4_11_label" tabindex="0">
Monitor
<span class="md-nav__icon md-icon"></span>
</label>
<nav class="md-nav" data-md-level="2" aria-labelledby="__nav_4_11_label" aria-expanded="false">
<label class="md-nav__title" for="__nav_4_11">
<span class="md-nav__icon md-icon"></span>
Monitor
</label>
<ul class="md-nav__list" data-md-scrollfix>
<li class="md-nav__item">
<a href="../../../monitor/overview/" class="md-nav__link">
Overview
</a>
</li>
<li class="md-nav__item">
<a href="../../../monitor/filesource/" class="md-nav__link">
filesource
</a>
</li>
<li class="md-nav__item">
<a href="../../../monitor/filewatcher/" class="md-nav__link">
filewatcher
</a>
</li>
<li class="md-nav__item">
<a href="../../../monitor/reload/" class="md-nav__link">
reload
</a>
</li>
<li class="md-nav__item">
<a href="../../../monitor/sink/" class="md-nav__link">
sink
</a>
</li>
<li class="md-nav__item">
<a href="../../../monitor/queue/" class="md-nav__link">
queue
</a>
</li>
<li class="md-nav__item">
<a href="../../../monitor/logalert/" class="md-nav__link">
logAlert
</a>
</li>
<li class="md-nav__item">
<a href="../../../monitor/sys/" class="md-nav__link">
sys
</a>
</li>
<li class="md-nav__item">
<a href="../../../monitor/info/" class="md-nav__link">
info
</a>
</li>
</ul>
</nav>
</li>
<li class="md-nav__item md-nav__item--section md-nav__item--nested">
<input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_4_12" >
<label class="md-nav__link" for="__nav_4_12" id="__nav_4_12_label" tabindex="0">
APIs
<span class="md-nav__icon md-icon"></span>
</label>
<nav class="md-nav" data-md-level="2" aria-labelledby="__nav_4_12_label" aria-expanded="false">
<label class="md-nav__title" for="__nav_4_12">
<span class="md-nav__icon md-icon"></span>
APIs
</label>
<ul class="md-nav__list" data-md-scrollfix>
<li class="md-nav__item">
<a href="../../../apis/help/" class="md-nav__link">
help
</a>
</li>
<li class="md-nav__item">
<a href="../../../apis/version/" class="md-nav__link">
version
</a>
</li>
<li class="md-nav__item">
<a href="../../../apis/config/" class="md-nav__link">
config
</a>
</li>
<li class="md-nav__item">
<a href="../../../apis/registry/" class="md-nav__link">
registry
</a>
</li>
</ul>
</nav>
</li>
</ul>
</nav>
</li>
<li class="md-nav__item md-nav__item--nested">
<input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_5" >
<label class="md-nav__link" for="__nav_5" id="__nav_5_label" tabindex="0">
开发手册
<span class="md-nav__icon md-icon"></span>
</label>
<nav class="md-nav" data-md-level="1" aria-labelledby="__nav_5_label" aria-expanded="false">
<label class="md-nav__title" for="__nav_5">
<span class="md-nav__icon md-icon"></span>
开发手册
</label>
<ul class="md-nav__list" data-md-scrollfix>
<li class="md-nav__item">
<a href="../../../../developer-guide/contributing/" class="md-nav__link">
代码贡献
</a>
</li>
<li class="md-nav__item">
<a href="../../../../developer-guide/build/" class="md-nav__link">
编译构建
</a>
</li>
<li class="md-nav__item">
<a href="../../../../developer-guide/development/" class="md-nav__link">
本地开发
</a>
</li>
<li class="md-nav__item">
<a href="../../../../developer-guide/code/coding-guide/" class="md-nav__link">
代码规范
</a>
</li>
<li class="md-nav__item">
<a href="../../../../developer-guide/component/component-guide/" class="md-nav__link">
组件开发
</a>
</li>
<li class="md-nav__item">
<a href="../../../../developer-guide/release/" class="md-nav__link">
版本发布流程
</a>
</li>
</ul>
</nav>
</li>
<li class="md-nav__item md-nav__item--nested">
<input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_6" >
<label class="md-nav__link" for="__nav_6" id="__nav_6_label" tabindex="0">
Blog
<span class="md-nav__icon md-icon"></span>
</label>
<nav class="md-nav" data-md-level="1" aria-labelledby="__nav_6_label" aria-expanded="false">
<label class="md-nav__title" for="__nav_6">
<span class="md-nav__icon md-icon"></span>
Blog
</label>
<ul class="md-nav__list" data-md-scrollfix>
<li class="md-nav__item">
<a href="../../../../blog/" class="md-nav__link">
Posts
</a>
</li>
</ul>
</nav>
</li>
</ul>
</nav>
</div>
</div>
</div>
<div class="md-sidebar md-sidebar--secondary" data-md-component="sidebar" data-md-type="toc" >
<div class="md-sidebar__scrollwrap">
<div class="md-sidebar__inner">
<nav class="md-nav md-nav--secondary" aria-label="目录">
<label class="md-nav__title" for="__toc">
<span class="md-nav__icon md-icon"></span>
目录
</label>
<ul class="md-nav__list" data-md-component="toc" data-md-scrollfix>
<li class="md-nav__item">
<a href="#brokers" class="md-nav__link">
brokers
</a>
</li>
<li class="md-nav__item">
<a href="#topic" class="md-nav__link">
topic
</a>
</li>
<li class="md-nav__item">
<a href="#topics" class="md-nav__link">
topics
</a>
</li>
<li class="md-nav__item">
<a href="#groupid" class="md-nav__link">
groupId
</a>
</li>
<li class="md-nav__item">
<a href="#clientid" class="md-nav__link">
clientId
</a>
</li>
<li class="md-nav__item">
<a href="#worker" class="md-nav__link">
worker
</a>
</li>
<li class="md-nav__item">
<a href="#addonmeta" class="md-nav__link">
addonMeta
</a>
</li>
<li class="md-nav__item">
<a href="#queuecapacity" class="md-nav__link">
queueCapacity
</a>
</li>
<li class="md-nav__item">
<a href="#minacceptedbytes" class="md-nav__link">
minAcceptedBytes
</a>
</li>
<li class="md-nav__item">
<a href="#maxacceptedbytes" class="md-nav__link">
maxAcceptedBytes
</a>
</li>
<li class="md-nav__item">
<a href="#readmaxattempts" class="md-nav__link">
readMaxAttempts
</a>
</li>
<li class="md-nav__item">
<a href="#maxpollwait" class="md-nav__link">
maxPollWait
</a>
</li>
<li class="md-nav__item">
<a href="#readbackoffmin" class="md-nav__link">
readBackoffMin
</a>
</li>
<li class="md-nav__item">
<a href="#readbackoffmax" class="md-nav__link">
readBackoffMax
</a>
</li>
<li class="md-nav__item">
<a href="#enableautocommit" class="md-nav__link">
enableAutoCommit
</a>
</li>
<li class="md-nav__item">
<a href="#autocommitinterval" class="md-nav__link">
autoCommitInterval
</a>
</li>
<li class="md-nav__item">
<a href="#autooffsetreset" class="md-nav__link">
autoOffsetReset
</a>
</li>
<li class="md-nav__item">
<a href="#sasl" class="md-nav__link">
sasl
</a>
</li>
</ul>
</nav>
</div>
</div>
</div>
<div class="md-content" data-md-component="content">
<article class="md-content__inner md-typeset">
<a href="https://github.com/loggie-io/docs/tree/main/docs/reference/pipelines/source/kafka.md" title="编辑此页" class="md-content__button md-icon">
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M10 20H6V4h7v5h5v3.1l2-2V8l-6-6H6c-1.1 0-2 .9-2 2v16c0 1.1.9 2 2 2h4v-2m10.2-7c.1 0 .3.1.4.2l1.3 1.3c.2.2.2.6 0 .8l-1 1-2.1-2.1 1-1c.1-.1.2-.2.4-.2m0 3.9L14.1 23H12v-2.1l6.1-6.1 2.1 2.1Z"/></svg>
</a>
<h1 id="kafka">kafka<a class="headerlink" href="#kafka" title="Permanent link">&para;</a></h1>
<p>Kafka source用于接收Kafka数据。</p>
<div class="admonition example">
<p class="admonition-title">Example</p>
<div class="highlight"><pre><span></span><code><span class="nt">sources</span><span class="p">:</span>
<span class="p p-Indicator">-</span><span class="w"> </span><span class="nt">type</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">kafka</span>
<span class="w"> </span><span class="nt">brokers</span><span class="p">:</span><span class="w"> </span><span class="p p-Indicator">[</span><span class="s">&quot;kafka1.kafka.svc:9092&quot;</span><span class="p p-Indicator">]</span>
<span class="w"> </span><span class="nt">topic</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">log-*</span>
</code></pre></div>
</div>
<div class="admonition note">
<p class="admonition-title">支持的Kafka版本</p>
<p>该kafka sink使用的是<a href="https://github.com/segmentio/kafka-go">segmentio/kafka-go</a>库。当前Loggie使用的库版本为<code>v0.4.39</code>对应版本测试支持的Kafka版本为<a href="https://github.com/segmentio/kafka-go/tree/v0.4.39#kafka-versions">0.10.1.0 - 2.7.1</a></p>
</div>
<h2 id="brokers">brokers<a class="headerlink" href="#brokers" title="Permanent link">&para;</a></h2>
<table>
<thead>
<tr>
<th><code>字段</code></th>
<th><code>类型</code></th>
<th><code>是否必填</code></th>
<th><code>默认值</code></th>
<th><code>含义</code></th>
</tr>
</thead>
<tbody>
<tr>
<td>brokers</td>
<td>string数组</td>
<td>必填</td>
<td></td>
<td>Kafka broker地址</td>
</tr>
</tbody>
</table>
<h2 id="topic">topic<a class="headerlink" href="#topic" title="Permanent link">&para;</a></h2>
<table>
<thead>
<tr>
<th><code>字段</code></th>
<th><code>类型</code></th>
<th><code>是否必填</code></th>
<th><code>默认值</code></th>
<th><code>含义</code></th>
</tr>
</thead>
<tbody>
<tr>
<td>topic</td>
<td>string</td>
<td>必填</td>
<td></td>
<td>接收的topic可使用正则来匹配多个topic</td>
</tr>
</tbody>
</table>
<h2 id="topics">topics<a class="headerlink" href="#topics" title="Permanent link">&para;</a></h2>
<table>
<thead>
<tr>
<th><code>字段</code></th>
<th><code>类型</code></th>
<th><code>是否必填</code></th>
<th><code>默认值</code></th>
<th><code>含义</code></th>
</tr>
</thead>
<tbody>
<tr>
<td>topics</td>
<td>数组</td>
<td>必填</td>
<td></td>
<td>接收的topics可填多个topic或者使用正则来匹配多个topic</td>
</tr>
</tbody>
</table>
<h2 id="groupid">groupId<a class="headerlink" href="#groupid" title="Permanent link">&para;</a></h2>
<table>
<thead>
<tr>
<th><code>字段</code></th>
<th><code>类型</code></th>
<th><code>是否必填</code></th>
<th><code>默认值</code></th>
<th><code>含义</code></th>
</tr>
</thead>
<tbody>
<tr>
<td>groupId</td>
<td>string</td>
<td>非必填</td>
<td>loggie</td>
<td>Loggie消费kafka的groupId</td>
</tr>
</tbody>
</table>
<h2 id="clientid">clientId<a class="headerlink" href="#clientid" title="Permanent link">&para;</a></h2>
<table>
<thead>
<tr>
<th><code>字段</code></th>
<th><code>类型</code></th>
<th><code>是否必填</code></th>
<th><code>默认值</code></th>
<th><code>含义</code></th>
</tr>
</thead>
<tbody>
<tr>
<td>clientId</td>
<td>string</td>
<td>非必填</td>
<td>loggie</td>
<td>Loggie消费kafka的clientId</td>
</tr>
</tbody>
</table>
<h2 id="worker">worker<a class="headerlink" href="#worker" title="Permanent link">&para;</a></h2>
<table>
<thead>
<tr>
<th><code>字段</code></th>
<th><code>类型</code></th>
<th><code>是否必填</code></th>
<th><code>默认值</code></th>
<th><code>含义</code></th>
</tr>
</thead>
<tbody>
<tr>
<td>worker</td>
<td>int</td>
<td>非必填</td>
<td>1</td>
<td>Loggie消费Kafka的worker线程个数</td>
</tr>
</tbody>
</table>
<h2 id="addonmeta">addonMeta<a class="headerlink" href="#addonmeta" title="Permanent link">&para;</a></h2>
<table>
<thead>
<tr>
<th><code>字段</code></th>
<th><code>类型</code></th>
<th><code>是否必填</code></th>
<th><code>默认值</code></th>
<th><code>含义</code></th>
</tr>
</thead>
<tbody>
<tr>
<td>addonMeta</td>
<td>bool</td>
<td>非必填</td>
<td>true</td>
<td>是否增加消费Kafka的一些元信息</td>
</tr>
</tbody>
</table>
<p>增加的元信息字段包括:</p>
<ul>
<li>offset: 消费的偏移量</li>
<li>partition: 分区</li>
<li>timestamp: 消费时打上的时间戳格式为RFC3339"2006-01-02T15:04:05Z07:00"</li>
<li>topic: 消费的topic</li>
</ul>
<h2 id="queuecapacity">queueCapacity<a class="headerlink" href="#queuecapacity" title="Permanent link">&para;</a></h2>
<table>
<thead>
<tr>
<th><code>字段</code></th>
<th><code>类型</code></th>
<th><code>是否必填</code></th>
<th><code>默认值</code></th>
<th><code>含义</code></th>
</tr>
</thead>
<tbody>
<tr>
<td>queueCapacity</td>
<td>int</td>
<td>非必填</td>
<td>100</td>
<td>内部发送的队列容量</td>
</tr>
</tbody>
</table>
<h2 id="minacceptedbytes">minAcceptedBytes<a class="headerlink" href="#minacceptedbytes" title="Permanent link">&para;</a></h2>
<table>
<thead>
<tr>
<th><code>字段</code></th>
<th><code>类型</code></th>
<th><code>是否必填</code></th>
<th><code>默认值</code></th>
<th><code>含义</code></th>
</tr>
</thead>
<tbody>
<tr>
<td>minAcceptedBytes</td>
<td>int</td>
<td>非必填</td>
<td>1</td>
<td>最小接收的batch字节数</td>
</tr>
</tbody>
</table>
<h2 id="maxacceptedbytes">maxAcceptedBytes<a class="headerlink" href="#maxacceptedbytes" title="Permanent link">&para;</a></h2>
<table>
<thead>
<tr>
<th><code>字段</code></th>
<th><code>类型</code></th>
<th><code>是否必填</code></th>
<th><code>默认值</code></th>
<th><code>含义</code></th>
</tr>
</thead>
<tbody>
<tr>
<td>maxAcceptedBytes</td>
<td>int</td>
<td>非必填</td>
<td>1e61MB)</td>
<td>最大接收的消息字节数如果超过会被truncate可以设置为一个能容忍的较大的值</td>
</tr>
</tbody>
</table>
<h2 id="readmaxattempts">readMaxAttempts<a class="headerlink" href="#readmaxattempts" title="Permanent link">&para;</a></h2>
<table>
<thead>
<tr>
<th><code>字段</code></th>
<th><code>类型</code></th>
<th><code>是否必填</code></th>
<th><code>默认值</code></th>
<th><code>含义</code></th>
</tr>
</thead>
<tbody>
<tr>
<td>readMaxAttempts</td>
<td>int</td>
<td>非必填</td>
<td>3</td>
<td>最大的重试次数</td>
</tr>
</tbody>
</table>
<h2 id="maxpollwait">maxPollWait<a class="headerlink" href="#maxpollwait" title="Permanent link">&para;</a></h2>
<table>
<thead>
<tr>
<th><code>字段</code></th>
<th><code>类型</code></th>
<th><code>是否必填</code></th>
<th><code>默认值</code></th>
<th><code>含义</code></th>
</tr>
</thead>
<tbody>
<tr>
<td>maxPollWait</td>
<td>time.Duration</td>
<td>非必填</td>
<td>10s</td>
<td>接收的最长等待时间</td>
</tr>
</tbody>
</table>
<h2 id="readbackoffmin">readBackoffMin<a class="headerlink" href="#readbackoffmin" title="Permanent link">&para;</a></h2>
<table>
<thead>
<tr>
<th><code>字段</code></th>
<th><code>类型</code></th>
<th><code>是否必填</code></th>
<th><code>默认值</code></th>
<th><code>含义</code></th>
</tr>
</thead>
<tbody>
<tr>
<td>readBackoffMin</td>
<td>time.Duration</td>
<td>非必填</td>
<td>100ms</td>
<td>在接收新的消息前,最小的时间间隔</td>
</tr>
</tbody>
</table>
<h2 id="readbackoffmax">readBackoffMax<a class="headerlink" href="#readbackoffmax" title="Permanent link">&para;</a></h2>
<table>
<thead>
<tr>
<th><code>字段</code></th>
<th><code>类型</code></th>
<th><code>是否必填</code></th>
<th><code>默认值</code></th>
<th><code>含义</code></th>
</tr>
</thead>
<tbody>
<tr>
<td>readBackoffMax</td>
<td>time.Duration</td>
<td>非必填</td>
<td>1s</td>
<td>在接收新的消息前,最大的时间间隔</td>
</tr>
</tbody>
</table>
<h2 id="enableautocommit">enableAutoCommit<a class="headerlink" href="#enableautocommit" title="Permanent link">&para;</a></h2>
<table>
<thead>
<tr>
<th><code>字段</code></th>
<th><code>类型</code></th>
<th><code>是否必填</code></th>
<th><code>默认值</code></th>
<th><code>含义</code></th>
</tr>
</thead>
<tbody>
<tr>
<td>enableAutoCommit</td>
<td>bool</td>
<td>非必填</td>
<td>false</td>
<td>是否开启autoCommit</td>
</tr>
</tbody>
</table>
<h2 id="autocommitinterval">autoCommitInterval<a class="headerlink" href="#autocommitinterval" title="Permanent link">&para;</a></h2>
<table>
<thead>
<tr>
<th><code>字段</code></th>
<th><code>类型</code></th>
<th><code>是否必填</code></th>
<th><code>默认值</code></th>
<th><code>含义</code></th>
</tr>
</thead>
<tbody>
<tr>
<td>autoCommitInterval</td>
<td>time.Duration</td>
<td>非必填</td>
<td>1s</td>
<td>autoCommit的间隔时间</td>
</tr>
</tbody>
</table>
<h2 id="autooffsetreset">autoOffsetReset<a class="headerlink" href="#autooffsetreset" title="Permanent link">&para;</a></h2>
<table>
<thead>
<tr>
<th><code>字段</code></th>
<th><code>类型</code></th>
<th><code>是否必填</code></th>
<th><code>默认值</code></th>
<th><code>含义</code></th>
</tr>
</thead>
<tbody>
<tr>
<td>autoOffsetReset</td>
<td>string</td>
<td>非必填</td>
<td>latest</td>
<td>没有offset时初始的offset采用方式可为<code>earliest</code><code>latest</code></td>
</tr>
</tbody>
</table>
<h2 id="sasl">sasl<a class="headerlink" href="#sasl" title="Permanent link">&para;</a></h2>
<table>
<thead>
<tr>
<th><code>字段</code></th>
<th><code>类型</code></th>
<th><code>是否必填</code></th>
<th><code>默认值</code></th>
<th><code>含义</code></th>
</tr>
</thead>
<tbody>
<tr>
<td>sasl</td>
<td></td>
<td>非必填</td>
<td></td>
<td>SASL authentication</td>
</tr>
<tr>
<td>sasl.type</td>
<td>string</td>
<td>必填</td>
<td></td>
<td>SASL类型可为<code>plain</code><code>scram</code></td>
</tr>
<tr>
<td>sasl.usename</td>
<td>string</td>
<td>必填</td>
<td></td>
<td>用户名请注意在v1.4和之前使用的名称为<code>userName</code>,当然后续版本也对此进行了兼容</td>
</tr>
<tr>
<td>sasl.password</td>
<td>string</td>
<td>必填</td>
<td></td>
<td>密码</td>
</tr>
<tr>
<td>sasl.algorithm</td>
<td>string</td>
<td>type=scram时必填</td>
<td></td>
<td>type=scram时使用的算法可选<code>sha256</code><code>sha512</code></td>
</tr>
</tbody>
</table>
</article>
</div>
<script>var tabs=__md_get("__tabs");if(Array.isArray(tabs))e:for(var set of document.querySelectorAll(".tabbed-set")){var tab,labels=set.querySelector(".tabbed-labels");for(tab of tabs)for(var label of labels.getElementsByTagName("label"))if(label.innerText.trim()===tab){var input=document.getElementById(label.htmlFor);input.checked=!0;continue e}}</script>
</div>
<button type="button" class="md-top md-icon" data-md-component="top" hidden>
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M13 20h-2V8l-5.5 5.5-1.42-1.42L12 4.16l7.92 7.92-1.42 1.42L13 8v12Z"/></svg>
回到页面顶部
</button>
</main>
<footer class="md-footer">
<div class="md-footer-meta md-typeset">
<div class="md-footer-meta__inner md-grid">
<div class="md-copyright">
<div class="md-copyright__highlight">
Copyright &copy; 2022 Loggie.io
</div>
Made with
<a href="https://squidfunk.github.io/mkdocs-material/" target="_blank" rel="noopener">
Material for MkDocs
</a>
</div>
<div class="md-social">
<a href="https://github.com/loggie-io" target="_blank" rel="noopener" title="github.com" class="md-social__link">
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 496 512"><!--! Font Awesome Free 6.4.0 by @fontawesome - https://fontawesome.com License - https://fontawesome.com/license/free (Icons: CC BY 4.0, Fonts: SIL OFL 1.1, Code: MIT License) Copyright 2023 Fonticons, Inc.--><path d="M165.9 397.4c0 2-2.3 3.6-5.2 3.6-3.3.3-5.6-1.3-5.6-3.6 0-2 2.3-3.6 5.2-3.6 3-.3 5.6 1.3 5.6 3.6zm-31.1-4.5c-.7 2 1.3 4.3 4.3 4.9 2.6 1 5.6 0 6.2-2s-1.3-4.3-4.3-5.2c-2.6-.7-5.5.3-6.2 2.3zm44.2-1.7c-2.9.7-4.9 2.6-4.6 4.9.3 2 2.9 3.3 5.9 2.6 2.9-.7 4.9-2.6 4.6-4.6-.3-1.9-3-3.2-5.9-2.9zM244.8 8C106.1 8 0 113.3 0 252c0 110.9 69.8 205.8 169.5 239.2 12.8 2.3 17.3-5.6 17.3-12.1 0-6.2-.3-40.4-.3-61.4 0 0-70 15-84.7-29.8 0 0-11.4-29.1-27.8-36.6 0 0-22.9-15.7 1.6-15.4 0 0 24.9 2 38.6 25.8 21.9 38.6 58.6 27.5 72.9 20.9 2.3-16 8.8-27.1 16-33.7-55.9-6.2-112.3-14.3-112.3-110.5 0-27.5 7.6-41.3 23.6-58.9-2.6-6.5-11.1-33.3 2.6-67.9 20.9-6.5 69 27 69 27 20-5.6 41.5-8.5 62.8-8.5s42.8 2.9 62.8 8.5c0 0 48.1-33.6 69-27 13.7 34.7 5.2 61.4 2.6 67.9 16 17.7 25.8 31.5 25.8 58.9 0 96.5-58.9 104.2-114.8 110.5 9.2 7.9 17 22.9 17 46.4 0 33.7-.3 75.4-.3 83.6 0 6.5 4.6 14.4 17.3 12.1C428.2 457.8 496 362.9 496 252 496 113.3 383.5 8 244.8 8zM97.2 352.9c-1.3 1-1 3.3.7 5.2 1.6 1.6 3.9 2.3 5.2 1 1.3-1 1-3.3-.7-5.2-1.6-1.6-3.9-2.3-5.2-1zm-10.8-8.1c-.7 1.3.3 2.9 2.3 3.9 1.6 1 3.6.7 4.3-.7.7-1.3-.3-2.9-2.3-3.9-2-.6-3.6-.3-4.3.7zm32.4 35.6c-1.6 1.3-1 4.3 1.3 6.2 2.3 2.3 5.2 2.6 6.5 1 1.3-1.3.7-4.3-1.3-6.2-2.2-2.3-5.2-2.6-6.5-1zm-11.4-14.7c-1.6 1-1.6 3.6 0 5.9 1.6 2.3 4.3 3.3 5.6 2.3 1.6-1.3 1.6-3.9 0-6.2-1.4-2.3-4-3.3-5.6-2z"/></svg>
</a>
<a href="https://hub.docker.com/r/loggieio/loggie" target="_blank" rel="noopener" title="hub.docker.com" class="md-social__link">
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 640 512"><!--! Font Awesome Free 6.4.0 by @fontawesome - https://fontawesome.com License - https://fontawesome.com/license/free (Icons: CC BY 4.0, Fonts: SIL OFL 1.1, Code: MIT License) Copyright 2023 Fonticons, Inc.--><path d="M349.9 236.3h-66.1v-59.4h66.1v59.4zm0-204.3h-66.1v60.7h66.1V32zm78.2 144.8H362v59.4h66.1v-59.4zm-156.3-72.1h-66.1v60.1h66.1v-60.1zm78.1 0h-66.1v60.1h66.1v-60.1zm276.8 100c-14.4-9.7-47.6-13.2-73.1-8.4-3.3-24-16.7-44.9-41.1-63.7l-14-9.3-9.3 14c-18.4 27.8-23.4 73.6-3.7 103.8-8.7 4.7-25.8 11.1-48.4 10.7H2.4c-8.7 50.8 5.8 116.8 44 162.1 37.1 43.9 92.7 66.2 165.4 66.2 157.4 0 273.9-72.5 328.4-204.2 21.4.4 67.6.1 91.3-45.2 1.5-2.5 6.6-13.2 8.5-17.1l-13.3-8.9zm-511.1-27.9h-66v59.4h66.1v-59.4zm78.1 0h-66.1v59.4h66.1v-59.4zm78.1 0h-66.1v59.4h66.1v-59.4zm-78.1-72.1h-66.1v60.1h66.1v-60.1z"/></svg>
</a>
</div>
</div>
</div>
</footer>
</div>
<div class="md-dialog" data-md-component="dialog">
<div class="md-dialog__inner md-typeset"></div>
</div>
<script id="__config" type="application/json">{"base": "../../../..", "features": ["content.code.annotate", "content.tabs.link", "navigation.instant", "navigation.sections", "search.highlight", "search.share", "search.suggest", "navigation.tabs", "navigation.tracking", "navigation.tabs.sticky", "navigation.top", "content.code.copy", "content.action.edit"], "search": "../../../../assets/javascripts/workers/search.74e28a9f.min.js", "translations": {"clipboard.copied": "\u5df2\u590d\u5236", "clipboard.copy": "\u590d\u5236", "search.result.more.one": "\u5728\u8be5\u9875\u4e0a\u8fd8\u6709 1 \u4e2a\u7b26\u5408\u6761\u4ef6\u7684\u7ed3\u679c", "search.result.more.other": "\u5728\u8be5\u9875\u4e0a\u8fd8\u6709 # \u4e2a\u7b26\u5408\u6761\u4ef6\u7684\u7ed3\u679c", "search.result.none": "\u6ca1\u6709\u627e\u5230\u7b26\u5408\u6761\u4ef6\u7684\u7ed3\u679c", "search.result.one": "\u627e\u5230 1 \u4e2a\u7b26\u5408\u6761\u4ef6\u7684\u7ed3\u679c", "search.result.other": "# \u4e2a\u7b26\u5408\u6761\u4ef6\u7684\u7ed3\u679c", "search.result.placeholder": "\u952e\u5165\u4ee5\u5f00\u59cb\u641c\u7d22", "search.result.term.missing": "\u7f3a\u5c11", "select.version": "\u9009\u62e9\u5f53\u524d\u7248\u672c"}, "version": {"provider": "mike"}}</script>
<script src="../../../../assets/javascripts/bundle.220ee61c.min.js"></script>
</body>
</html>