docs/user-guide/best-practice/log-enrich/index.html

2865 lines
77 KiB
HTML
Raw Blame History

This file contains ambiguous Unicode characters

This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.

<!doctype html>
<html lang="zh" class="no-js">
<head>
<meta charset="utf-8">
<meta name="viewport" content="width=device-width,initial-scale=1">
<meta name="description" content="Loggie Documentation">
<link rel="prev" href="../../use-in-kubernetes/kube-event-source/">
<link rel="next" href="../log-process/">
<link rel="icon" href="../../../assets/min-logo.png">
<meta name="generator" content="mkdocs-1.4.3, mkdocs-material-9.1.18">
<title>日志格式与元信息字段 - </title>
<link rel="stylesheet" href="../../../assets/stylesheets/main.26e3688c.min.css">
<link rel="preconnect" href="https://fonts.gstatic.com" crossorigin>
<link rel="stylesheet" href="https://fonts.googleapis.com/css?family=Roboto:300,300i,400,400i,700,700i%7CRobot+Mono:400,400i,700,700i&display=fallback">
<style>:root{--md-text-font:"Roboto";--md-code-font:"Robot Mono"}</style>
<link rel="stylesheet" href="../../../stylesheets/extra.css">
<script>__md_scope=new URL("../../..",location),__md_hash=e=>[...e].reduce((e,_)=>(e<<5)-e+_.charCodeAt(0),0),__md_get=(e,_=localStorage,t=__md_scope)=>JSON.parse(_.getItem(t.pathname+"."+e)),__md_set=(e,_,t=localStorage,a=__md_scope)=>{try{t.setItem(a.pathname+"."+e,JSON.stringify(_))}catch(e){}}</script>
</head>
<body dir="ltr">
<script>var palette=__md_get("__palette");if(palette&&"object"==typeof palette.color)for(var key of Object.keys(palette.color))document.body.setAttribute("data-md-color-"+key,palette.color[key])</script>
<input class="md-toggle" data-md-toggle="drawer" type="checkbox" id="__drawer" autocomplete="off">
<input class="md-toggle" data-md-toggle="search" type="checkbox" id="__search" autocomplete="off">
<label class="md-overlay" for="__drawer"></label>
<div data-md-component="skip">
<a href="#_1" class="md-skip">
跳转至
</a>
</div>
<div data-md-component="announce">
</div>
<div data-md-color-scheme="default" data-md-component="outdated" hidden>
</div>
<header class="md-header md-header--shadow md-header--lifted" data-md-component="header">
<nav class="md-header__inner md-grid" aria-label="页眉">
<a href="../../.." title=" " class="md-header__button md-logo" aria-label=" " data-md-component="logo">
<img src="../../../assets/loggie.svg" alt="logo">
</a>
<label class="md-header__button md-icon" for="__drawer">
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M3 6h18v2H3V6m0 5h18v2H3v-2m0 5h18v2H3v-2Z"/></svg>
</label>
<div class="md-header__title" data-md-component="header-title">
<div class="md-header__ellipsis">
<div class="md-header__topic">
<span class="md-ellipsis">
</span>
</div>
<div class="md-header__topic" data-md-component="header-topic">
<span class="md-ellipsis">
日志格式与元信息字段
</span>
</div>
</div>
</div>
<div class="md-header__option">
<div class="md-select">
<button class="md-header__button md-icon" aria-label="选择当前语言">
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="m12.87 15.07-2.54-2.51.03-.03A17.52 17.52 0 0 0 14.07 6H17V4h-7V2H8v2H1v2h11.17C11.5 7.92 10.44 9.75 9 11.35 8.07 10.32 7.3 9.19 6.69 8h-2c.73 1.63 1.73 3.17 2.98 4.56l-5.09 5.02L4 19l5-5 3.11 3.11.76-2.04M18.5 10h-2L12 22h2l1.12-3h4.75L21 22h2l-4.5-12m-2.62 7 1.62-4.33L19.12 17h-3.24Z"/></svg>
</button>
<div class="md-select__inner">
<ul class="md-select__list">
<li class="md-select__item">
<a href="https://loggie-io.github.io/docs-en/" hreflang="en" class="md-select__link">
English
</a>
</li>
</ul>
</div>
</div>
</div>
<label class="md-header__button md-icon" for="__search">
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M9.5 3A6.5 6.5 0 0 1 16 9.5c0 1.61-.59 3.09-1.56 4.23l.27.27h.79l5 5-1.5 1.5-5-5v-.79l-.27-.27A6.516 6.516 0 0 1 9.5 16 6.5 6.5 0 0 1 3 9.5 6.5 6.5 0 0 1 9.5 3m0 2C7 5 5 7 5 9.5S7 14 9.5 14 14 12 14 9.5 12 5 9.5 5Z"/></svg>
</label>
<div class="md-search" data-md-component="search" role="dialog">
<label class="md-search__overlay" for="__search"></label>
<div class="md-search__inner" role="search">
<form class="md-search__form" name="search">
<input type="text" class="md-search__input" name="query" aria-label="搜索" placeholder="搜索" autocapitalize="off" autocorrect="off" autocomplete="off" spellcheck="false" data-md-component="search-query" required>
<label class="md-search__icon md-icon" for="__search">
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M9.5 3A6.5 6.5 0 0 1 16 9.5c0 1.61-.59 3.09-1.56 4.23l.27.27h.79l5 5-1.5 1.5-5-5v-.79l-.27-.27A6.516 6.516 0 0 1 9.5 16 6.5 6.5 0 0 1 3 9.5 6.5 6.5 0 0 1 9.5 3m0 2C7 5 5 7 5 9.5S7 14 9.5 14 14 12 14 9.5 12 5 9.5 5Z"/></svg>
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M20 11v2H8l5.5 5.5-1.42 1.42L4.16 12l7.92-7.92L13.5 5.5 8 11h12Z"/></svg>
</label>
<nav class="md-search__options" aria-label="查找">
<a href="javascript:void(0)" class="md-search__icon md-icon" title="分享" aria-label="分享" data-clipboard data-clipboard-text="" data-md-component="search-share" tabindex="-1">
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M18 16.08c-.76 0-1.44.3-1.96.77L8.91 12.7c.05-.23.09-.46.09-.7 0-.24-.04-.47-.09-.7l7.05-4.11c.54.5 1.25.81 2.04.81a3 3 0 0 0 3-3 3 3 0 0 0-3-3 3 3 0 0 0-3 3c0 .24.04.47.09.7L8.04 9.81C7.5 9.31 6.79 9 6 9a3 3 0 0 0-3 3 3 3 0 0 0 3 3c.79 0 1.5-.31 2.04-.81l7.12 4.15c-.05.21-.08.43-.08.66 0 1.61 1.31 2.91 2.92 2.91 1.61 0 2.92-1.3 2.92-2.91A2.92 2.92 0 0 0 18 16.08Z"/></svg>
</a>
<button type="reset" class="md-search__icon md-icon" title="清空当前内容" aria-label="清空当前内容" tabindex="-1">
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M19 6.41 17.59 5 12 10.59 6.41 5 5 6.41 10.59 12 5 17.59 6.41 19 12 13.41 17.59 19 19 17.59 13.41 12 19 6.41Z"/></svg>
</button>
</nav>
<div class="md-search__suggest" data-md-component="search-suggest"></div>
</form>
<div class="md-search__output">
<div class="md-search__scrollwrap" data-md-scrollfix>
<div class="md-search-result" data-md-component="search-result">
<div class="md-search-result__meta">
正在初始化搜索引擎
</div>
<ol class="md-search-result__list" role="presentation"></ol>
</div>
</div>
</div>
</div>
</div>
<div class="md-header__source">
<a href="https://github.com/loggie-io/loggie" title="前往仓库" class="md-source" data-md-component="source">
<div class="md-source__icon md-icon">
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 496 512"><!--! Font Awesome Free 6.4.0 by @fontawesome - https://fontawesome.com License - https://fontawesome.com/license/free (Icons: CC BY 4.0, Fonts: SIL OFL 1.1, Code: MIT License) Copyright 2023 Fonticons, Inc.--><path d="M165.9 397.4c0 2-2.3 3.6-5.2 3.6-3.3.3-5.6-1.3-5.6-3.6 0-2 2.3-3.6 5.2-3.6 3-.3 5.6 1.3 5.6 3.6zm-31.1-4.5c-.7 2 1.3 4.3 4.3 4.9 2.6 1 5.6 0 6.2-2s-1.3-4.3-4.3-5.2c-2.6-.7-5.5.3-6.2 2.3zm44.2-1.7c-2.9.7-4.9 2.6-4.6 4.9.3 2 2.9 3.3 5.9 2.6 2.9-.7 4.9-2.6 4.6-4.6-.3-1.9-3-3.2-5.9-2.9zM244.8 8C106.1 8 0 113.3 0 252c0 110.9 69.8 205.8 169.5 239.2 12.8 2.3 17.3-5.6 17.3-12.1 0-6.2-.3-40.4-.3-61.4 0 0-70 15-84.7-29.8 0 0-11.4-29.1-27.8-36.6 0 0-22.9-15.7 1.6-15.4 0 0 24.9 2 38.6 25.8 21.9 38.6 58.6 27.5 72.9 20.9 2.3-16 8.8-27.1 16-33.7-55.9-6.2-112.3-14.3-112.3-110.5 0-27.5 7.6-41.3 23.6-58.9-2.6-6.5-11.1-33.3 2.6-67.9 20.9-6.5 69 27 69 27 20-5.6 41.5-8.5 62.8-8.5s42.8 2.9 62.8 8.5c0 0 48.1-33.6 69-27 13.7 34.7 5.2 61.4 2.6 67.9 16 17.7 25.8 31.5 25.8 58.9 0 96.5-58.9 104.2-114.8 110.5 9.2 7.9 17 22.9 17 46.4 0 33.7-.3 75.4-.3 83.6 0 6.5 4.6 14.4 17.3 12.1C428.2 457.8 496 362.9 496 252 496 113.3 383.5 8 244.8 8zM97.2 352.9c-1.3 1-1 3.3.7 5.2 1.6 1.6 3.9 2.3 5.2 1 1.3-1 1-3.3-.7-5.2-1.6-1.6-3.9-2.3-5.2-1zm-10.8-8.1c-.7 1.3.3 2.9 2.3 3.9 1.6 1 3.6.7 4.3-.7.7-1.3-.3-2.9-2.3-3.9-2-.6-3.6-.3-4.3.7zm32.4 35.6c-1.6 1.3-1 4.3 1.3 6.2 2.3 2.3 5.2 2.6 6.5 1 1.3-1.3.7-4.3-1.3-6.2-2.2-2.3-5.2-2.6-6.5-1zm-11.4-14.7c-1.6 1-1.6 3.6 0 5.9 1.6 2.3 4.3 3.3 5.6 2.3 1.6-1.3 1.6-3.9 0-6.2-1.4-2.3-4-3.3-5.6-2z"/></svg>
</div>
<div class="md-source__repository">
loggie-io/loggie
</div>
</a>
</div>
</nav>
<nav class="md-tabs" aria-label="标签" data-md-component="tabs">
<div class="md-grid">
<ul class="md-tabs__list">
<li class="md-tabs__item">
<a href="../../.." class="md-tabs__link">
Home
</a>
</li>
<li class="md-tabs__item">
<a href="../../../getting-started/overview/" class="md-tabs__link">
快速上手
</a>
</li>
<li class="md-tabs__item">
<a href="../../" class="md-tabs__link md-tabs__link--active">
用户指南
</a>
</li>
<li class="md-tabs__item">
<a href="../../../reference/" class="md-tabs__link">
组件配置
</a>
</li>
<li class="md-tabs__item">
<a href="../../../developer-guide/contributing/" class="md-tabs__link">
开发手册
</a>
</li>
<li class="md-tabs__item">
<a href="../../../blog/" class="md-tabs__link">
Blog
</a>
</li>
</ul>
</div>
</nav>
</header>
<div class="md-container" data-md-component="container">
<main class="md-main" data-md-component="main">
<div class="md-main__inner md-grid">
<div class="md-sidebar md-sidebar--primary" data-md-component="sidebar" data-md-type="navigation" >
<div class="md-sidebar__scrollwrap">
<div class="md-sidebar__inner">
<nav class="md-nav md-nav--primary md-nav--lifted" aria-label="导航栏" data-md-level="0">
<label class="md-nav__title" for="__drawer">
<a href="../../.." title=" " class="md-nav__button md-logo" aria-label=" " data-md-component="logo">
<img src="../../../assets/loggie.svg" alt="logo">
</a>
</label>
<div class="md-nav__source">
<a href="https://github.com/loggie-io/loggie" title="前往仓库" class="md-source" data-md-component="source">
<div class="md-source__icon md-icon">
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 496 512"><!--! Font Awesome Free 6.4.0 by @fontawesome - https://fontawesome.com License - https://fontawesome.com/license/free (Icons: CC BY 4.0, Fonts: SIL OFL 1.1, Code: MIT License) Copyright 2023 Fonticons, Inc.--><path d="M165.9 397.4c0 2-2.3 3.6-5.2 3.6-3.3.3-5.6-1.3-5.6-3.6 0-2 2.3-3.6 5.2-3.6 3-.3 5.6 1.3 5.6 3.6zm-31.1-4.5c-.7 2 1.3 4.3 4.3 4.9 2.6 1 5.6 0 6.2-2s-1.3-4.3-4.3-5.2c-2.6-.7-5.5.3-6.2 2.3zm44.2-1.7c-2.9.7-4.9 2.6-4.6 4.9.3 2 2.9 3.3 5.9 2.6 2.9-.7 4.9-2.6 4.6-4.6-.3-1.9-3-3.2-5.9-2.9zM244.8 8C106.1 8 0 113.3 0 252c0 110.9 69.8 205.8 169.5 239.2 12.8 2.3 17.3-5.6 17.3-12.1 0-6.2-.3-40.4-.3-61.4 0 0-70 15-84.7-29.8 0 0-11.4-29.1-27.8-36.6 0 0-22.9-15.7 1.6-15.4 0 0 24.9 2 38.6 25.8 21.9 38.6 58.6 27.5 72.9 20.9 2.3-16 8.8-27.1 16-33.7-55.9-6.2-112.3-14.3-112.3-110.5 0-27.5 7.6-41.3 23.6-58.9-2.6-6.5-11.1-33.3 2.6-67.9 20.9-6.5 69 27 69 27 20-5.6 41.5-8.5 62.8-8.5s42.8 2.9 62.8 8.5c0 0 48.1-33.6 69-27 13.7 34.7 5.2 61.4 2.6 67.9 16 17.7 25.8 31.5 25.8 58.9 0 96.5-58.9 104.2-114.8 110.5 9.2 7.9 17 22.9 17 46.4 0 33.7-.3 75.4-.3 83.6 0 6.5 4.6 14.4 17.3 12.1C428.2 457.8 496 362.9 496 252 496 113.3 383.5 8 244.8 8zM97.2 352.9c-1.3 1-1 3.3.7 5.2 1.6 1.6 3.9 2.3 5.2 1 1.3-1 1-3.3-.7-5.2-1.6-1.6-3.9-2.3-5.2-1zm-10.8-8.1c-.7 1.3.3 2.9 2.3 3.9 1.6 1 3.6.7 4.3-.7.7-1.3-.3-2.9-2.3-3.9-2-.6-3.6-.3-4.3.7zm32.4 35.6c-1.6 1.3-1 4.3 1.3 6.2 2.3 2.3 5.2 2.6 6.5 1 1.3-1.3.7-4.3-1.3-6.2-2.2-2.3-5.2-2.6-6.5-1zm-11.4-14.7c-1.6 1-1.6 3.6 0 5.9 1.6 2.3 4.3 3.3 5.6 2.3 1.6-1.3 1.6-3.9 0-6.2-1.4-2.3-4-3.3-5.6-2z"/></svg>
</div>
<div class="md-source__repository">
loggie-io/loggie
</div>
</a>
</div>
<ul class="md-nav__list" data-md-scrollfix>
<li class="md-nav__item">
<a href="../../.." class="md-nav__link">
Home
</a>
</li>
<li class="md-nav__item md-nav__item--nested">
<input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_2" >
<label class="md-nav__link" for="__nav_2" id="__nav_2_label" tabindex="0">
快速上手
<span class="md-nav__icon md-icon"></span>
</label>
<nav class="md-nav" data-md-level="1" aria-labelledby="__nav_2_label" aria-expanded="false">
<label class="md-nav__title" for="__nav_2">
<span class="md-nav__icon md-icon"></span>
快速上手
</label>
<ul class="md-nav__list" data-md-scrollfix>
<li class="md-nav__item">
<a href="../../../getting-started/overview/" class="md-nav__link">
Overview
</a>
</li>
<li class="md-nav__item">
<a href="../../../getting-started/intro/core-concept/" class="md-nav__link">
核心概念
</a>
</li>
<li class="md-nav__item md-nav__item--section md-nav__item--nested">
<input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_2_3" >
<label class="md-nav__link" for="__nav_2_3" id="__nav_2_3_label" tabindex="0">
快速上手
<span class="md-nav__icon md-icon"></span>
</label>
<nav class="md-nav" data-md-level="2" aria-labelledby="__nav_2_3_label" aria-expanded="false">
<label class="md-nav__title" for="__nav_2_3">
<span class="md-nav__icon md-icon"></span>
快速上手
</label>
<ul class="md-nav__list" data-md-scrollfix>
<li class="md-nav__item">
<a href="../../../getting-started/quick-start/quick-start/" class="md-nav__link">
Overview
</a>
</li>
<li class="md-nav__item">
<a href="../../../getting-started/quick-start/kubernetes/" class="md-nav__link">
Kubernetes环境
</a>
</li>
<li class="md-nav__item">
<a href="../../../getting-started/quick-start/node/" class="md-nav__link">
主机环境
</a>
</li>
</ul>
</nav>
</li>
<li class="md-nav__item md-nav__item--section md-nav__item--nested">
<input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_2_4" >
<label class="md-nav__link" for="__nav_2_4" id="__nav_2_4_label" tabindex="0">
部署
<span class="md-nav__icon md-icon"></span>
</label>
<nav class="md-nav" data-md-level="2" aria-labelledby="__nav_2_4_label" aria-expanded="false">
<label class="md-nav__title" for="__nav_2_4">
<span class="md-nav__icon md-icon"></span>
部署
</label>
<ul class="md-nav__list" data-md-scrollfix>
<li class="md-nav__item">
<a href="../../../getting-started/install/kubernetes/" class="md-nav__link">
Kubernetes部署
</a>
</li>
<li class="md-nav__item">
<a href="../../../getting-started/install/node/" class="md-nav__link">
主机部署
</a>
</li>
</ul>
</nav>
</li>
<li class="md-nav__item md-nav__item--section md-nav__item--nested">
<input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_2_5" >
<label class="md-nav__link" for="__nav_2_5" id="__nav_2_5_label" tabindex="0">
RoadMap
<span class="md-nav__icon md-icon"></span>
</label>
<nav class="md-nav" data-md-level="2" aria-labelledby="__nav_2_5_label" aria-expanded="false">
<label class="md-nav__title" for="__nav_2_5">
<span class="md-nav__icon md-icon"></span>
RoadMap
</label>
<ul class="md-nav__list" data-md-scrollfix>
<li class="md-nav__item">
<a href="../../../getting-started/roadmap/roadmap-2023/" class="md-nav__link">
2023
</a>
</li>
</ul>
</nav>
</li>
</ul>
</nav>
</li>
<li class="md-nav__item md-nav__item--active md-nav__item--nested">
<input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_3" checked>
<label class="md-nav__link" for="__nav_3" id="__nav_3_label" tabindex="0">
用户指南
<span class="md-nav__icon md-icon"></span>
</label>
<nav class="md-nav" data-md-level="1" aria-labelledby="__nav_3_label" aria-expanded="true">
<label class="md-nav__title" for="__nav_3">
<span class="md-nav__icon md-icon"></span>
用户指南
</label>
<ul class="md-nav__list" data-md-scrollfix>
<li class="md-nav__item">
<a href="../../" class="md-nav__link">
Overview
</a>
</li>
<li class="md-nav__item md-nav__item--section md-nav__item--nested">
<input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_3_2" >
<label class="md-nav__link" for="__nav_3_2" id="__nav_3_2_label" tabindex="0">
架构与特性
<span class="md-nav__icon md-icon"></span>
</label>
<nav class="md-nav" data-md-level="2" aria-labelledby="__nav_3_2_label" aria-expanded="false">
<label class="md-nav__title" for="__nav_3_2">
<span class="md-nav__icon md-icon"></span>
架构与特性
</label>
<ul class="md-nav__list" data-md-scrollfix>
<li class="md-nav__item">
<a href="../../architecture/background/" class="md-nav__link">
诞生背景
</a>
</li>
<li class="md-nav__item">
<a href="../../architecture/core-arch/" class="md-nav__link">
设计架构
</a>
</li>
<li class="md-nav__item">
<a href="../../architecture/schema/" class="md-nav__link">
数据格式
</a>
</li>
<li class="md-nav__item">
<a href="../../architecture/advantages/" class="md-nav__link">
优势与特性
</a>
</li>
<li class="md-nav__item">
<a href="../../architecture/compare/" class="md-nav__link">
开源项目对比
</a>
</li>
</ul>
</nav>
</li>
<li class="md-nav__item md-nav__item--section md-nav__item--nested">
<input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_3_3" >
<label class="md-nav__link" for="__nav_3_3" id="__nav_3_3_label" tabindex="0">
Kubernetes
<span class="md-nav__icon md-icon"></span>
</label>
<nav class="md-nav" data-md-level="2" aria-labelledby="__nav_3_3_label" aria-expanded="false">
<label class="md-nav__title" for="__nav_3_3">
<span class="md-nav__icon md-icon"></span>
Kubernetes
</label>
<ul class="md-nav__list" data-md-scrollfix>
<li class="md-nav__item">
<a href="../../use-in-kubernetes/general-usage/" class="md-nav__link">
Kubernetes下的日志采集
</a>
</li>
<li class="md-nav__item">
<a href="../../use-in-kubernetes/collect-container-logs/" class="md-nav__link">
Loggie采集容器日志
</a>
</li>
<li class="md-nav__item">
<a href="../../use-in-kubernetes/collect-node-logs/" class="md-nav__link">
Loggie采集Node日志
</a>
</li>
<li class="md-nav__item">
<a href="../../use-in-kubernetes/sidecar/" class="md-nav__link">
Sidecar方式采集日志
</a>
</li>
<li class="md-nav__item">
<a href="../../use-in-kubernetes/kube-event-source/" class="md-nav__link">
采集Kubernetes Events
</a>
</li>
</ul>
</nav>
</li>
<li class="md-nav__item md-nav__item--active md-nav__item--section md-nav__item--nested">
<input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_3_4" checked>
<label class="md-nav__link" for="__nav_3_4" id="__nav_3_4_label" tabindex="0">
最佳实践
<span class="md-nav__icon md-icon"></span>
</label>
<nav class="md-nav" data-md-level="2" aria-labelledby="__nav_3_4_label" aria-expanded="true">
<label class="md-nav__title" for="__nav_3_4">
<span class="md-nav__icon md-icon"></span>
最佳实践
</label>
<ul class="md-nav__list" data-md-scrollfix>
<li class="md-nav__item md-nav__item--active">
<input class="md-nav__toggle md-toggle" type="checkbox" id="__toc">
<label class="md-nav__link md-nav__link--active" for="__toc">
日志格式与元信息字段
<span class="md-nav__icon md-icon"></span>
</label>
<a href="./" class="md-nav__link md-nav__link--active">
日志格式与元信息字段
</a>
<nav class="md-nav md-nav--secondary" aria-label="目录">
<label class="md-nav__title" for="__toc">
<span class="md-nav__icon md-icon"></span>
目录
</label>
<ul class="md-nav__list" data-md-component="toc" data-md-scrollfix>
<li class="md-nav__item">
<a href="#_2" class="md-nav__link">
字段格式转换
</a>
<nav class="md-nav" aria-label="字段格式转换">
<ul class="md-nav__list">
<li class="md-nav__item">
<a href="#schema-interceptor" class="md-nav__link">
使用schema interceptor
</a>
</li>
<li class="md-nav__item">
<a href="#transformer-interceptor" class="md-nav__link">
使用transformer interceptor
</a>
</li>
</ul>
</nav>
</li>
<li class="md-nav__item">
<a href="#_3" class="md-nav__link">
添加元信息
</a>
<nav class="md-nav" aria-label="添加元信息">
<ul class="md-nav__list">
<li class="md-nav__item">
<a href="#fields" class="md-nav__link">
添加fields自定义元信息
</a>
</li>
<li class="md-nav__item">
<a href="#file-source" class="md-nav__link">
添加日志采集file source的状态信息
</a>
</li>
<li class="md-nav__item">
<a href="#kubernetes" class="md-nav__link">
增加Kubernetes元信息
</a>
</li>
<li class="md-nav__item">
<a href="#meta" class="md-nav__link">
添加meta系统内置元信息
</a>
</li>
</ul>
</nav>
</li>
</ul>
</nav>
</li>
<li class="md-nav__item">
<a href="../log-process/" class="md-nav__link">
日志切分处理
</a>
</li>
<li class="md-nav__item">
<a href="../aggregator/" class="md-nav__link">
使用Loggie中转机
</a>
</li>
<li class="md-nav__item">
<a href="../concurrency/" class="md-nav__link">
自适应sink流量控制
</a>
</li>
</ul>
</nav>
</li>
<li class="md-nav__item md-nav__item--section md-nav__item--nested">
<input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_3_5" >
<label class="md-nav__link" for="__nav_3_5" id="__nav_3_5_label" tabindex="0">
监控报警
<span class="md-nav__icon md-icon"></span>
</label>
<nav class="md-nav" data-md-level="2" aria-labelledby="__nav_3_5_label" aria-expanded="false">
<label class="md-nav__title" for="__nav_3_5">
<span class="md-nav__icon md-icon"></span>
监控报警
</label>
<ul class="md-nav__list" data-md-scrollfix>
<li class="md-nav__item">
<a href="../../monitor/loggie-monitor/" class="md-nav__link">
Loggie的监控与报警
</a>
</li>
<li class="md-nav__item">
<a href="../../monitor/service-log-alarm/" class="md-nav__link">
日志报警
</a>
</li>
</ul>
</nav>
</li>
<li class="md-nav__item md-nav__item--section md-nav__item--nested">
<input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_3_6" >
<label class="md-nav__link" for="__nav_3_6" id="__nav_3_6_label" tabindex="0">
企业实战
<span class="md-nav__icon md-icon"></span>
</label>
<nav class="md-nav" data-md-level="2" aria-labelledby="__nav_3_6_label" aria-expanded="false">
<label class="md-nav__title" for="__nav_3_6">
<span class="md-nav__icon md-icon"></span>
企业实战
</label>
<ul class="md-nav__list" data-md-scrollfix>
<li class="md-nav__item">
<a href="../../enterprise-practice/architecture-and-evolution/" class="md-nav__link">
日志系统架构与演进
</a>
</li>
<li class="md-nav__item">
<a href="../../enterprise-practice/sls/" class="md-nav__link">
使用阿里云可观测统一存储SLS
</a>
</li>
</ul>
</nav>
</li>
<li class="md-nav__item md-nav__item--section md-nav__item--nested">
<input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_3_7" >
<label class="md-nav__link" for="__nav_3_7" id="__nav_3_7_label" tabindex="0">
运维排障
<span class="md-nav__icon md-icon"></span>
</label>
<nav class="md-nav" data-md-level="2" aria-labelledby="__nav_3_7_label" aria-expanded="false">
<label class="md-nav__title" for="__nav_3_7">
<span class="md-nav__icon md-icon"></span>
运维排障
</label>
<ul class="md-nav__list" data-md-scrollfix>
<li class="md-nav__item">
<a href="../../troubleshot/log-collection/" class="md-nav__link">
日志采集快速排障指南
</a>
</li>
<li class="md-nav__item">
<a href="../../troubleshot/problems/" class="md-nav__link">
问题案例
</a>
</li>
</ul>
</nav>
</li>
</ul>
</nav>
</li>
<li class="md-nav__item md-nav__item--nested">
<input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_4" >
<label class="md-nav__link" for="__nav_4" id="__nav_4_label" tabindex="0">
组件配置
<span class="md-nav__icon md-icon"></span>
</label>
<nav class="md-nav" data-md-level="1" aria-labelledby="__nav_4_label" aria-expanded="false">
<label class="md-nav__title" for="__nav_4">
<span class="md-nav__icon md-icon"></span>
组件配置
</label>
<ul class="md-nav__list" data-md-scrollfix>
<li class="md-nav__item">
<a href="../../../reference/" class="md-nav__link">
Overview
</a>
</li>
<li class="md-nav__item">
<a href="../../../reference/global/args/" class="md-nav__link">
启动参数
</a>
</li>
<li class="md-nav__item">
<a href="../../../reference/global/var/" class="md-nav__link">
字段变量
</a>
</li>
<li class="md-nav__item md-nav__item--section md-nav__item--nested">
<input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_4_4" >
<label class="md-nav__link" for="__nav_4_4" id="__nav_4_4_label" tabindex="0">
系统配置
<span class="md-nav__icon md-icon"></span>
</label>
<nav class="md-nav" data-md-level="2" aria-labelledby="__nav_4_4_label" aria-expanded="false">
<label class="md-nav__title" for="__nav_4_4">
<span class="md-nav__icon md-icon"></span>
系统配置
</label>
<ul class="md-nav__list" data-md-scrollfix>
<li class="md-nav__item">
<a href="../../../reference/global/monitor/" class="md-nav__link">
monitor
</a>
</li>
<li class="md-nav__item">
<a href="../../../reference/global/discovery/" class="md-nav__link">
discovery
</a>
</li>
<li class="md-nav__item">
<a href="../../../reference/global/reload/" class="md-nav__link">
reload
</a>
</li>
<li class="md-nav__item">
<a href="../../../reference/global/defaults/" class="md-nav__link">
defaults
</a>
</li>
<li class="md-nav__item">
<a href="../../../reference/global/http/" class="md-nav__link">
http
</a>
</li>
</ul>
</nav>
</li>
<li class="md-nav__item md-nav__item--section md-nav__item--nested">
<input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_4_5" >
<label class="md-nav__link" for="__nav_4_5" id="__nav_4_5_label" tabindex="0">
Kubernetes CRD
<span class="md-nav__icon md-icon"></span>
</label>
<nav class="md-nav" data-md-level="2" aria-labelledby="__nav_4_5_label" aria-expanded="false">
<label class="md-nav__title" for="__nav_4_5">
<span class="md-nav__icon md-icon"></span>
Kubernetes CRD
</label>
<ul class="md-nav__list" data-md-scrollfix>
<li class="md-nav__item">
<a href="../../../reference/discovery/kubernetes/logconfig/" class="md-nav__link">
LogConfig
</a>
</li>
<li class="md-nav__item">
<a href="../../../reference/discovery/kubernetes/clusterlogconfig/" class="md-nav__link">
ClusterLogConfig
</a>
</li>
<li class="md-nav__item">
<a href="../../../reference/discovery/kubernetes/sink/" class="md-nav__link">
Sink
</a>
</li>
<li class="md-nav__item">
<a href="../../../reference/discovery/kubernetes/interceptors/" class="md-nav__link">
Interceptor
</a>
</li>
</ul>
</nav>
</li>
<li class="md-nav__item md-nav__item--section md-nav__item--nested">
<input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_4_6" >
<label class="md-nav__link" for="__nav_4_6" id="__nav_4_6_label" tabindex="0">
Source
<span class="md-nav__icon md-icon"></span>
</label>
<nav class="md-nav" data-md-level="2" aria-labelledby="__nav_4_6_label" aria-expanded="false">
<label class="md-nav__title" for="__nav_4_6">
<span class="md-nav__icon md-icon"></span>
Source
</label>
<ul class="md-nav__list" data-md-scrollfix>
<li class="md-nav__item">
<a href="../../../reference/pipelines/source/overview/" class="md-nav__link">
Overview
</a>
</li>
<li class="md-nav__item">
<a href="../../../reference/pipelines/source/file/" class="md-nav__link">
file
</a>
</li>
<li class="md-nav__item">
<a href="../../../reference/pipelines/source/kafka/" class="md-nav__link">
kafka
</a>
</li>
<li class="md-nav__item">
<a href="../../../reference/pipelines/source/elasticsearch/" class="md-nav__link">
elasticsearch
</a>
</li>
<li class="md-nav__item">
<a href="../../../reference/pipelines/source/kube-event/" class="md-nav__link">
kubeEvent
</a>
</li>
<li class="md-nav__item">
<a href="../../../reference/pipelines/source/grpc/" class="md-nav__link">
grpc
</a>
</li>
<li class="md-nav__item">
<a href="../../../reference/pipelines/source/prometheus-exporter/" class="md-nav__link">
prometheusExporter
</a>
</li>
<li class="md-nav__item">
<a href="../../../reference/pipelines/source/unix/" class="md-nav__link">
unix
</a>
</li>
<li class="md-nav__item">
<a href="../../../reference/pipelines/source/dev/" class="md-nav__link">
dev
</a>
</li>
</ul>
</nav>
</li>
<li class="md-nav__item md-nav__item--section md-nav__item--nested">
<input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_4_7" >
<label class="md-nav__link" for="__nav_4_7" id="__nav_4_7_label" tabindex="0">
Sink
<span class="md-nav__icon md-icon"></span>
</label>
<nav class="md-nav" data-md-level="2" aria-labelledby="__nav_4_7_label" aria-expanded="false">
<label class="md-nav__title" for="__nav_4_7">
<span class="md-nav__icon md-icon"></span>
Sink
</label>
<ul class="md-nav__list" data-md-scrollfix>
<li class="md-nav__item">
<a href="../../../reference/pipelines/sink/overview/" class="md-nav__link">
Overview
</a>
</li>
<li class="md-nav__item">
<a href="../../../reference/pipelines/sink/elasticsearch/" class="md-nav__link">
elasticsearch
</a>
</li>
<li class="md-nav__item">
<a href="../../../reference/pipelines/sink/kafka/" class="md-nav__link">
kafka
</a>
</li>
<li class="md-nav__item">
<a href="../../../reference/pipelines/sink/franzkafka/" class="md-nav__link">
kafka(franz)
</a>
</li>
<li class="md-nav__item">
<a href="../../../reference/pipelines/sink/loki/" class="md-nav__link">
loki
</a>
</li>
<li class="md-nav__item">
<a href="../../../reference/pipelines/sink/pulsar/" class="md-nav__link">
pulsar
</a>
</li>
<li class="md-nav__item">
<a href="../../../reference/pipelines/sink/grpc/" class="md-nav__link">
grpc
</a>
</li>
<li class="md-nav__item">
<a href="../../../reference/pipelines/sink/file/" class="md-nav__link">
file
</a>
</li>
<li class="md-nav__item">
<a href="../../../reference/pipelines/sink/dev/" class="md-nav__link">
dev
</a>
</li>
<li class="md-nav__item">
<a href="../../../reference/pipelines/sink/sls/" class="md-nav__link">
sls
</a>
</li>
<li class="md-nav__item">
<a href="../../../reference/pipelines/sink/webhook/" class="md-nav__link">
alertwebhook
</a>
</li>
<li class="md-nav__item">
<a href="../../../reference/pipelines/sink/zinc/" class="md-nav__link">
zinc
</a>
</li>
</ul>
</nav>
</li>
<li class="md-nav__item md-nav__item--section md-nav__item--nested">
<input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_4_8" >
<label class="md-nav__link" for="__nav_4_8" id="__nav_4_8_label" tabindex="0">
Interceptor
<span class="md-nav__icon md-icon"></span>
</label>
<nav class="md-nav" data-md-level="2" aria-labelledby="__nav_4_8_label" aria-expanded="false">
<label class="md-nav__title" for="__nav_4_8">
<span class="md-nav__icon md-icon"></span>
Interceptor
</label>
<ul class="md-nav__list" data-md-scrollfix>
<li class="md-nav__item">
<a href="../../../reference/pipelines/interceptor/overview/" class="md-nav__link">
Overview
</a>
</li>
<li class="md-nav__item">
<a href="../../../reference/pipelines/interceptor/schema/" class="md-nav__link">
schema
</a>
</li>
<li class="md-nav__item">
<a href="../../../reference/pipelines/interceptor/transformer/" class="md-nav__link">
transformer
</a>
</li>
<li class="md-nav__item">
<a href="../../../reference/pipelines/interceptor/limit/" class="md-nav__link">
rateLimit
</a>
</li>
<li class="md-nav__item">
<a href="../../../reference/pipelines/interceptor/addk8smeta/" class="md-nav__link">
addK8sMeta
</a>
</li>
<li class="md-nav__item">
<a href="../../../reference/pipelines/interceptor/logalert/" class="md-nav__link">
logAlert
</a>
</li>
<li class="md-nav__item">
<a href="../../../reference/pipelines/interceptor/metrics/" class="md-nav__link">
metrics
</a>
</li>
<li class="md-nav__item">
<a href="../../../reference/pipelines/interceptor/retry/" class="md-nav__link">
retry
</a>
</li>
<li class="md-nav__item">
<a href="../../../reference/pipelines/interceptor/maxbytes/" class="md-nav__link">
maxbytes
</a>
</li>
<li class="md-nav__item">
<a href="../../../reference/pipelines/interceptor/normalize/" class="md-nav__link">
normalize
</a>
</li>
</ul>
</nav>
</li>
<li class="md-nav__item md-nav__item--section md-nav__item--nested">
<input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_4_9" >
<label class="md-nav__link" for="__nav_4_9" id="__nav_4_9_label" tabindex="0">
Queue
<span class="md-nav__icon md-icon"></span>
</label>
<nav class="md-nav" data-md-level="2" aria-labelledby="__nav_4_9_label" aria-expanded="false">
<label class="md-nav__title" for="__nav_4_9">
<span class="md-nav__icon md-icon"></span>
Queue
</label>
<ul class="md-nav__list" data-md-scrollfix>
<li class="md-nav__item">
<a href="../../../reference/pipelines/queue/channel/" class="md-nav__link">
channel
</a>
</li>
<li class="md-nav__item">
<a href="../../../reference/pipelines/queue/memory/" class="md-nav__link">
memory
</a>
</li>
</ul>
</nav>
</li>
<li class="md-nav__item md-nav__item--section md-nav__item--nested">
<input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_4_10" >
<label class="md-nav__link" for="__nav_4_10" id="__nav_4_10_label" tabindex="0">
Monitor
<span class="md-nav__icon md-icon"></span>
</label>
<nav class="md-nav" data-md-level="2" aria-labelledby="__nav_4_10_label" aria-expanded="false">
<label class="md-nav__title" for="__nav_4_10">
<span class="md-nav__icon md-icon"></span>
Monitor
</label>
<ul class="md-nav__list" data-md-scrollfix>
<li class="md-nav__item">
<a href="../../../reference/monitor/overview/" class="md-nav__link">
Overview
</a>
</li>
<li class="md-nav__item">
<a href="../../../reference/monitor/filesource/" class="md-nav__link">
filesource
</a>
</li>
<li class="md-nav__item">
<a href="../../../reference/monitor/filewatcher/" class="md-nav__link">
filewatcher
</a>
</li>
<li class="md-nav__item">
<a href="../../../reference/monitor/reload/" class="md-nav__link">
reload
</a>
</li>
<li class="md-nav__item">
<a href="../../../reference/monitor/sink/" class="md-nav__link">
sink
</a>
</li>
<li class="md-nav__item">
<a href="../../../reference/monitor/queue/" class="md-nav__link">
queue
</a>
</li>
<li class="md-nav__item">
<a href="../../../reference/monitor/logalert/" class="md-nav__link">
logAlert
</a>
</li>
<li class="md-nav__item">
<a href="../../../reference/monitor/sys/" class="md-nav__link">
sys
</a>
</li>
<li class="md-nav__item">
<a href="../../../reference/monitor/info/" class="md-nav__link">
info
</a>
</li>
</ul>
</nav>
</li>
<li class="md-nav__item md-nav__item--section md-nav__item--nested">
<input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_4_11" >
<label class="md-nav__link" for="__nav_4_11" id="__nav_4_11_label" tabindex="0">
APIs
<span class="md-nav__icon md-icon"></span>
</label>
<nav class="md-nav" data-md-level="2" aria-labelledby="__nav_4_11_label" aria-expanded="false">
<label class="md-nav__title" for="__nav_4_11">
<span class="md-nav__icon md-icon"></span>
APIs
</label>
<ul class="md-nav__list" data-md-scrollfix>
<li class="md-nav__item">
<a href="../../../reference/apis/ops/" class="md-nav__link">
内部运维类
</a>
</li>
</ul>
</nav>
</li>
</ul>
</nav>
</li>
<li class="md-nav__item md-nav__item--nested">
<input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_5" >
<label class="md-nav__link" for="__nav_5" id="__nav_5_label" tabindex="0">
开发手册
<span class="md-nav__icon md-icon"></span>
</label>
<nav class="md-nav" data-md-level="1" aria-labelledby="__nav_5_label" aria-expanded="false">
<label class="md-nav__title" for="__nav_5">
<span class="md-nav__icon md-icon"></span>
开发手册
</label>
<ul class="md-nav__list" data-md-scrollfix>
<li class="md-nav__item">
<a href="../../../developer-guide/contributing/" class="md-nav__link">
代码贡献
</a>
</li>
<li class="md-nav__item">
<a href="../../../developer-guide/development/" class="md-nav__link">
本地开发
</a>
</li>
<li class="md-nav__item">
<a href="../../../developer-guide/code/coding-guide/" class="md-nav__link">
代码规范
</a>
</li>
<li class="md-nav__item">
<a href="../../../developer-guide/component/component-guide/" class="md-nav__link">
组件开发
</a>
</li>
<li class="md-nav__item">
<a href="../../../developer-guide/release/" class="md-nav__link">
版本发布流程
</a>
</li>
</ul>
</nav>
</li>
<li class="md-nav__item md-nav__item--nested">
<input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_6" >
<label class="md-nav__link" for="__nav_6" id="__nav_6_label" tabindex="0">
Blog
<span class="md-nav__icon md-icon"></span>
</label>
<nav class="md-nav" data-md-level="1" aria-labelledby="__nav_6_label" aria-expanded="false">
<label class="md-nav__title" for="__nav_6">
<span class="md-nav__icon md-icon"></span>
Blog
</label>
<ul class="md-nav__list" data-md-scrollfix>
<li class="md-nav__item">
<a href="../../../blog/" class="md-nav__link">
Posts
</a>
</li>
</ul>
</nav>
</li>
</ul>
</nav>
</div>
</div>
</div>
<div class="md-sidebar md-sidebar--secondary" data-md-component="sidebar" data-md-type="toc" >
<div class="md-sidebar__scrollwrap">
<div class="md-sidebar__inner">
<nav class="md-nav md-nav--secondary" aria-label="目录">
<label class="md-nav__title" for="__toc">
<span class="md-nav__icon md-icon"></span>
目录
</label>
<ul class="md-nav__list" data-md-component="toc" data-md-scrollfix>
<li class="md-nav__item">
<a href="#_2" class="md-nav__link">
字段格式转换
</a>
<nav class="md-nav" aria-label="字段格式转换">
<ul class="md-nav__list">
<li class="md-nav__item">
<a href="#schema-interceptor" class="md-nav__link">
使用schema interceptor
</a>
</li>
<li class="md-nav__item">
<a href="#transformer-interceptor" class="md-nav__link">
使用transformer interceptor
</a>
</li>
</ul>
</nav>
</li>
<li class="md-nav__item">
<a href="#_3" class="md-nav__link">
添加元信息
</a>
<nav class="md-nav" aria-label="添加元信息">
<ul class="md-nav__list">
<li class="md-nav__item">
<a href="#fields" class="md-nav__link">
添加fields自定义元信息
</a>
</li>
<li class="md-nav__item">
<a href="#file-source" class="md-nav__link">
添加日志采集file source的状态信息
</a>
</li>
<li class="md-nav__item">
<a href="#kubernetes" class="md-nav__link">
增加Kubernetes元信息
</a>
</li>
<li class="md-nav__item">
<a href="#meta" class="md-nav__link">
添加meta系统内置元信息
</a>
</li>
</ul>
</nav>
</li>
</ul>
</nav>
</div>
</div>
</div>
<div class="md-content" data-md-component="content">
<article class="md-content__inner md-typeset">
<h1 id="_1">日志格式与元信息字段<a class="headerlink" href="#_1" title="Permanent link">&para;</a></h1>
<blockquote>
<p>建议先了解Loggie内部日志数据<a href="../../architecture/schema/">schema设计</a></p>
</blockquote>
<p>Loggie部署在不同的环境中如果需要在原始的日志数据里增加一些元信息同时兼容已有的格式可以参考如下的办法。</p>
<h2 id="_2">字段格式转换<a class="headerlink" href="#_2" title="Permanent link">&para;</a></h2>
<h3 id="schema-interceptor">使用schema interceptor<a class="headerlink" href="#schema-interceptor" title="Permanent link">&para;</a></h3>
<p>使用schema interceptor可以增加时间字段以及pipelineName与sourceName字段。另外还可以对字段进行重命名比如修改<code>body</code><code>message</code><br />
请参考<a href="../../../reference/pipelines/interceptor/schema/">schema interceptor</a></p>
<p>由于大部分情况下我们需要全局生效而不是仅仅只在某个pipeline里添加该interceptor所以建议在系统配置的defaults中添加schema interceptor
这样可以避免每个pipeline均需配置该interceptor。 </p>
<div class="admonition config">
<p class="admonition-title">loggie.yml</p>
<div class="highlight"><pre><span></span><code><span class="nt">loggie</span><span class="p">:</span>
<span class="w"> </span><span class="nt">defaults</span><span class="p">:</span>
<span class="w"> </span><span class="nt">interceptors</span><span class="p">:</span>
<span class="w"> </span><span class="p p-Indicator">-</span><span class="w"> </span><span class="nt">type</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">schema</span>
<span class="w"> </span><span class="nt">name</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">global</span>
<span class="w"> </span><span class="nt">order</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">700</span>
<span class="w"> </span><span class="nt">addMeta</span><span class="p">:</span>
<span class="w"> </span><span class="nt">timestamp</span><span class="p">:</span>
<span class="w"> </span><span class="nt">key</span><span class="p">:</span><span class="w"> </span><span class="s">&quot;@timestamp&quot;</span>
<span class="w"> </span><span class="nt">remap</span><span class="p">:</span>
<span class="w"> </span><span class="nt">body</span><span class="p">:</span>
<span class="w"> </span><span class="nt">key</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">message</span>
</code></pre></div>
</div>
<p>这里的name是为了增加标识避免如果在pipeline中又新增schema interceptor会导致校验不通过。另外增加order字段为一个较小的值默认为900)这样default里的interceptor会优先于pipeline里定义的其他interceptor执行。</p>
<h3 id="transformer-interceptor">使用transformer interceptor<a class="headerlink" href="#transformer-interceptor" title="Permanent link">&para;</a></h3>
<p>tranformer提供了更丰富的功能可以应对复杂日志的场景。<br />
具体请参考<a href="../../../reference/pipelines/interceptor/transformer/">transformer interceptor</a></p>
<h2 id="_3">添加元信息<a class="headerlink" href="#_3" title="Permanent link">&para;</a></h2>
<h3 id="fields">添加fields自定义元信息<a class="headerlink" href="#fields" title="Permanent link">&para;</a></h3>
<p>如果我们在source上配置了一些自定义的fields。</p>
<div class="admonition config">
<p class="admonition-title">pipelines.yml</p>
<div class="highlight"><pre><span></span><code><span class="nt">pipelines</span><span class="p">:</span>
<span class="w"> </span><span class="p p-Indicator">-</span><span class="w"> </span><span class="nt">name</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">local</span>
<span class="w"> </span><span class="nt">sources</span><span class="p">:</span>
<span class="w"> </span><span class="p p-Indicator">-</span><span class="w"> </span><span class="nt">type</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">file</span>
<span class="w"> </span><span class="nt">name</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">demo</span>
<span class="w"> </span><span class="nt">paths</span><span class="p">:</span>
<span class="w"> </span><span class="p p-Indicator">-</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">/tmp/log/*.log</span>
<span class="w"> </span><span class="nt">fields</span><span class="p">:</span>
<span class="w"> </span><span class="nt">topic</span><span class="p">:</span><span class="w"> </span><span class="s">&quot;loggie&quot;</span>
<span class="w"> </span><span class="nt">sink</span><span class="p">:</span>
<span class="w"> </span><span class="nt">type</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">dev</span>
<span class="w"> </span><span class="nt">printEvents</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">true</span>
<span class="w"> </span><span class="nt">codec</span><span class="p">:</span>
<span class="w"> </span><span class="nt">pretty</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">true</span>
</code></pre></div>
</div>
<p>那么sink输出的为</p>
<div class="highlight"><pre><span></span><code><span class="p">{</span>
<span class="w"> </span><span class="nt">&quot;fields&quot;</span><span class="p">:</span><span class="w"> </span><span class="p">{</span>
<span class="w"> </span><span class="nt">&quot;topic&quot;</span><span class="p">:</span><span class="w"> </span><span class="s2">&quot;loggie&quot;</span><span class="p">,</span>
<span class="w"> </span><span class="p">},</span>
<span class="w"> </span><span class="nt">&quot;body&quot;</span><span class="p">:</span><span class="w"> </span><span class="s2">&quot;01-Dec-2021 03:13:58.298 INFO [main] Starting service [Catalina]&quot;</span>
<span class="p">}</span>
</code></pre></div>
<p>当然我们也可以配置<code>fieldsUnderRoot: true</code>让fields里的<code>key:value</code>和body同一层级。</p>
<div class="admonition config">
<p class="admonition-title">pipelines.yml</p>
<div class="highlight"><pre><span></span><code><span class="nt">pipelines</span><span class="p">:</span>
<span class="w"> </span><span class="p p-Indicator">-</span><span class="w"> </span><span class="nt">name</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">local</span>
<span class="w"> </span><span class="nt">sources</span><span class="p">:</span>
<span class="w"> </span><span class="p p-Indicator">-</span><span class="w"> </span><span class="nt">type</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">file</span>
<span class="w"> </span><span class="nt">fields</span><span class="p">:</span>
<span class="w"> </span><span class="nt">topic</span><span class="p">:</span><span class="w"> </span><span class="s">&quot;loggie&quot;</span>
<span class="w"> </span><span class="nt">fieldsUnderRoot</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">true</span>
<span class="nn">...</span>
</code></pre></div>
</div>
<div class="highlight"><pre><span></span><code><span class="p">{</span>
<span class="w"> </span><span class="nt">&quot;topic&quot;</span><span class="p">:</span><span class="w"> </span><span class="s2">&quot;loggie&quot;</span><span class="p">,</span>
<span class="w"> </span><span class="nt">&quot;body&quot;</span><span class="p">:</span><span class="w"> </span><span class="s2">&quot;01-Dec-2021 03:13:58.298 INFO [main] Starting service [Catalina]&quot;</span>
<span class="p">}</span>
</code></pre></div>
<h3 id="file-source">添加日志采集file source的状态信息<a class="headerlink" href="#file-source" title="Permanent link">&para;</a></h3>
<p>在我们使用file source时可能希望自动在日志原始数据里增加一些日志采集的状态比如采集的文件名称、采集的文件offsest等file source提供了一个<code>addonMeta</code>配置可快速enable。</p>
<p>示例:添加如下<code>addonMeta</code>并设置为true。</p>
<div class="admonition config">
<p class="admonition-title">file source</p>
<div class="highlight"><pre><span></span><code><span class="nt">sources</span><span class="p">:</span>
<span class="p p-Indicator">-</span><span class="w"> </span><span class="nt">type</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">file</span>
<span class="w"> </span><span class="nt">paths</span><span class="p">:</span>
<span class="w"> </span><span class="p p-Indicator">-</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">/var/log/*.log</span>
<span class="w"> </span><span class="nt">addonMeta</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">true</span>
</code></pre></div>
</div>
<p>此时采集的event会变成类似如下</p>
<div class="admonition example">
<p class="admonition-title">Example</p>
<div class="highlight"><pre><span></span><code><span class="p">{</span>
<span class="w"> </span><span class="nt">&quot;body&quot;</span><span class="p">:</span><span class="w"> </span><span class="s2">&quot;this is test&quot;</span><span class="p">,</span>
<span class="w"> </span><span class="nt">&quot;state&quot;</span><span class="p">:</span><span class="w"> </span><span class="p">{</span>
<span class="w"> </span><span class="nt">&quot;pipeline&quot;</span><span class="p">:</span><span class="w"> </span><span class="s2">&quot;local&quot;</span><span class="p">,</span>
<span class="w"> </span><span class="nt">&quot;source&quot;</span><span class="p">:</span><span class="w"> </span><span class="s2">&quot;demo&quot;</span><span class="p">,</span>
<span class="w"> </span><span class="nt">&quot;filename&quot;</span><span class="p">:</span><span class="w"> </span><span class="s2">&quot;/var/log/a.log&quot;</span><span class="p">,</span>
<span class="w"> </span><span class="nt">&quot;timestamp&quot;</span><span class="p">:</span><span class="w"> </span><span class="s2">&quot;2006-01-02T15:04:05.000Z&quot;</span><span class="p">,</span>
<span class="w"> </span><span class="nt">&quot;offset&quot;</span><span class="p">:</span><span class="w"> </span><span class="mi">1024</span><span class="p">,</span>
<span class="w"> </span><span class="nt">&quot;bytes&quot;</span><span class="p">:</span><span class="w"> </span><span class="mi">4096</span><span class="p">,</span>
<span class="w"> </span><span class="nt">&quot;hostname&quot;</span><span class="p">:</span><span class="w"> </span><span class="s2">&quot;node-1&quot;</span>
<span class="w"> </span><span class="p">}</span>
<span class="p">}</span>
</code></pre></div>
</div>
<p>具体字段含义可参考<a href="../../../reference/pipelines/source/file/">file source</a></p>
<h3 id="kubernetes">增加Kubernetes元信息<a class="headerlink" href="#kubernetes" title="Permanent link">&para;</a></h3>
<p>在Kubernetes的场景中采集的容器日志为了在查询的时候使用namespace/podName等信息进行检索往往需要增加相关的元数据。</p>
<p>我们可以在系统配置的discovery.kubernetes中配置额外的k8s fields字段。</p>
<p>可参考<a href="../../../reference/global/discovery/">discovery</a></p>
<h3 id="meta">添加meta系统内置元信息<a class="headerlink" href="#meta" title="Permanent link">&para;</a></h3>
<p>有一些Loggie系统内置的元信息我们也希望发送给下游这个时候需要使用normalize interceptor中的addMeta processors。
(需要注意的是,该操作会对采集传输性能有一定影响,正常情况下,并不建议使用该方式)</p>
<div class="admonition config">
<p class="admonition-title">pipelines.yml</p>
<div class="highlight"><pre><span></span><code><span class="nt">pipelines</span><span class="p">:</span>
<span class="w"> </span><span class="p p-Indicator">-</span><span class="w"> </span><span class="nt">name</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">local</span>
<span class="w"> </span><span class="nt">sources</span><span class="p">:</span>
<span class="w"> </span><span class="p p-Indicator">-</span><span class="w"> </span><span class="nt">type</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">file</span>
<span class="w"> </span><span class="nt">name</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">demo</span>
<span class="w"> </span><span class="nt">paths</span><span class="p">:</span>
<span class="w"> </span><span class="p p-Indicator">-</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">/tmp/log/*.log</span>
<span class="w"> </span><span class="nt">fields</span><span class="p">:</span>
<span class="w"> </span><span class="nt">topic</span><span class="p">:</span><span class="w"> </span><span class="s">&quot;loggie&quot;</span>
<span class="w"> </span><span class="nt">interceptors</span><span class="p">:</span>
<span class="w"> </span><span class="p p-Indicator">-</span><span class="w"> </span><span class="nt">type</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">normalize</span>
<span class="w"> </span><span class="nt">processors</span><span class="p">:</span>
<span class="w"> </span><span class="p p-Indicator">-</span><span class="w"> </span><span class="nt">addMeta</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">~</span>
<span class="w"> </span><span class="nt">sink</span><span class="p">:</span>
<span class="w"> </span><span class="nt">type</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">dev</span>
<span class="w"> </span><span class="nt">printEvents</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">true</span>
<span class="w"> </span><span class="nt">codec</span><span class="p">:</span>
<span class="w"> </span><span class="nt">pretty</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">true</span>
</code></pre></div>
</div>
<p>配置了addMeta processor之后默认会把所有的系统内置元信息输出。</p>
<p>默认Json格式输出示例如下</p>
<div class="admonition example">
<p class="admonition-title">Example</p>
<div class="highlight"><pre><span></span><code><span class="p">{</span>
<span class="w"> </span><span class="nt">&quot;fields&quot;</span><span class="p">:</span><span class="w"> </span><span class="p">{</span>
<span class="w"> </span><span class="nt">&quot;topic&quot;</span><span class="p">:</span><span class="w"> </span><span class="s2">&quot;loggie&quot;</span>
<span class="w"> </span><span class="p">},</span>
<span class="w"> </span><span class="nt">&quot;meta&quot;</span><span class="p">:</span><span class="w"> </span><span class="p">{</span>
<span class="w"> </span><span class="nt">&quot;systemState&quot;</span><span class="p">:</span><span class="w"> </span><span class="p">{</span>
<span class="w"> </span><span class="nt">&quot;nextOffset&quot;</span><span class="p">:</span><span class="w"> </span><span class="mi">720</span><span class="p">,</span>
<span class="w"> </span><span class="nt">&quot;filename&quot;</span><span class="p">:</span><span class="w"> </span><span class="s2">&quot;/tmp/log/a.log&quot;</span><span class="p">,</span>
<span class="w"> </span><span class="nt">&quot;collectTime&quot;</span><span class="p">:</span><span class="w"> </span><span class="s2">&quot;2022-03-08T11:33:47.369813+08:00&quot;</span><span class="p">,</span>
<span class="w"> </span><span class="nt">&quot;contentBytes&quot;</span><span class="p">:</span><span class="w"> </span><span class="mi">90</span><span class="p">,</span>
<span class="w"> </span><span class="nt">&quot;jobUid&quot;</span><span class="p">:</span><span class="w"> </span><span class="s2">&quot;43772050-16777231&quot;</span><span class="p">,</span>
<span class="w"> </span><span class="nt">&quot;lineNumber&quot;</span><span class="p">:</span><span class="w"> </span><span class="mi">8</span><span class="p">,</span>
<span class="w"> </span><span class="nt">&quot;offset&quot;</span><span class="p">:</span><span class="w"> </span><span class="mi">630</span>
<span class="w"> </span><span class="p">},</span>
<span class="w"> </span><span class="nt">&quot;systemProductTime&quot;</span><span class="p">:</span><span class="w"> </span><span class="s2">&quot;2022-03-08T11:33:47.370166+08:00&quot;</span><span class="p">,</span>
<span class="w"> </span><span class="nt">&quot;systemPipelineName&quot;</span><span class="p">:</span><span class="w"> </span><span class="s2">&quot;local&quot;</span><span class="p">,</span>
<span class="w"> </span><span class="nt">&quot;systemSourceName&quot;</span><span class="p">:</span><span class="w"> </span><span class="s2">&quot;demo&quot;</span>
<span class="w"> </span><span class="p">},</span>
<span class="w"> </span><span class="nt">&quot;body&quot;</span><span class="p">:</span><span class="w"> </span><span class="s2">&quot;01-Dec-2021 03:13:58.298 INFO [main] Starting service [Catalina]&quot;</span>
<span class="p">}</span>
</code></pre></div>
</div>
<p>当然,我们可能会觉得这些数据太多了,或者想对字段进行修改。我们就可以使用<a href="../../../reference/pipelines/interceptor/transformer/">transformer interceptor</a>里的action进行操作。 </p>
</article>
</div>
<script>var tabs=__md_get("__tabs");if(Array.isArray(tabs))e:for(var set of document.querySelectorAll(".tabbed-set")){var tab,labels=set.querySelector(".tabbed-labels");for(tab of tabs)for(var label of labels.getElementsByTagName("label"))if(label.innerText.trim()===tab){var input=document.getElementById(label.htmlFor);input.checked=!0;continue e}}</script>
</div>
<button type="button" class="md-top md-icon" data-md-component="top" hidden>
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M13 20h-2V8l-5.5 5.5-1.42-1.42L12 4.16l7.92 7.92-1.42 1.42L13 8v12Z"/></svg>
回到页面顶部
</button>
</main>
<footer class="md-footer">
<div class="md-footer-meta md-typeset">
<div class="md-footer-meta__inner md-grid">
<div class="md-copyright">
<div class="md-copyright__highlight">
Copyright &copy; 2022 Loggie.io
</div>
Made with
<a href="https://squidfunk.github.io/mkdocs-material/" target="_blank" rel="noopener">
Material for MkDocs
</a>
</div>
<div class="md-social">
<a href="https://github.com/loggie-io" target="_blank" rel="noopener" title="github.com" class="md-social__link">
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 496 512"><!--! Font Awesome Free 6.4.0 by @fontawesome - https://fontawesome.com License - https://fontawesome.com/license/free (Icons: CC BY 4.0, Fonts: SIL OFL 1.1, Code: MIT License) Copyright 2023 Fonticons, Inc.--><path d="M165.9 397.4c0 2-2.3 3.6-5.2 3.6-3.3.3-5.6-1.3-5.6-3.6 0-2 2.3-3.6 5.2-3.6 3-.3 5.6 1.3 5.6 3.6zm-31.1-4.5c-.7 2 1.3 4.3 4.3 4.9 2.6 1 5.6 0 6.2-2s-1.3-4.3-4.3-5.2c-2.6-.7-5.5.3-6.2 2.3zm44.2-1.7c-2.9.7-4.9 2.6-4.6 4.9.3 2 2.9 3.3 5.9 2.6 2.9-.7 4.9-2.6 4.6-4.6-.3-1.9-3-3.2-5.9-2.9zM244.8 8C106.1 8 0 113.3 0 252c0 110.9 69.8 205.8 169.5 239.2 12.8 2.3 17.3-5.6 17.3-12.1 0-6.2-.3-40.4-.3-61.4 0 0-70 15-84.7-29.8 0 0-11.4-29.1-27.8-36.6 0 0-22.9-15.7 1.6-15.4 0 0 24.9 2 38.6 25.8 21.9 38.6 58.6 27.5 72.9 20.9 2.3-16 8.8-27.1 16-33.7-55.9-6.2-112.3-14.3-112.3-110.5 0-27.5 7.6-41.3 23.6-58.9-2.6-6.5-11.1-33.3 2.6-67.9 20.9-6.5 69 27 69 27 20-5.6 41.5-8.5 62.8-8.5s42.8 2.9 62.8 8.5c0 0 48.1-33.6 69-27 13.7 34.7 5.2 61.4 2.6 67.9 16 17.7 25.8 31.5 25.8 58.9 0 96.5-58.9 104.2-114.8 110.5 9.2 7.9 17 22.9 17 46.4 0 33.7-.3 75.4-.3 83.6 0 6.5 4.6 14.4 17.3 12.1C428.2 457.8 496 362.9 496 252 496 113.3 383.5 8 244.8 8zM97.2 352.9c-1.3 1-1 3.3.7 5.2 1.6 1.6 3.9 2.3 5.2 1 1.3-1 1-3.3-.7-5.2-1.6-1.6-3.9-2.3-5.2-1zm-10.8-8.1c-.7 1.3.3 2.9 2.3 3.9 1.6 1 3.6.7 4.3-.7.7-1.3-.3-2.9-2.3-3.9-2-.6-3.6-.3-4.3.7zm32.4 35.6c-1.6 1.3-1 4.3 1.3 6.2 2.3 2.3 5.2 2.6 6.5 1 1.3-1.3.7-4.3-1.3-6.2-2.2-2.3-5.2-2.6-6.5-1zm-11.4-14.7c-1.6 1-1.6 3.6 0 5.9 1.6 2.3 4.3 3.3 5.6 2.3 1.6-1.3 1.6-3.9 0-6.2-1.4-2.3-4-3.3-5.6-2z"/></svg>
</a>
<a href="https://hub.docker.com/r/loggieio/loggie" target="_blank" rel="noopener" title="hub.docker.com" class="md-social__link">
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 640 512"><!--! Font Awesome Free 6.4.0 by @fontawesome - https://fontawesome.com License - https://fontawesome.com/license/free (Icons: CC BY 4.0, Fonts: SIL OFL 1.1, Code: MIT License) Copyright 2023 Fonticons, Inc.--><path d="M349.9 236.3h-66.1v-59.4h66.1v59.4zm0-204.3h-66.1v60.7h66.1V32zm78.2 144.8H362v59.4h66.1v-59.4zm-156.3-72.1h-66.1v60.1h66.1v-60.1zm78.1 0h-66.1v60.1h66.1v-60.1zm276.8 100c-14.4-9.7-47.6-13.2-73.1-8.4-3.3-24-16.7-44.9-41.1-63.7l-14-9.3-9.3 14c-18.4 27.8-23.4 73.6-3.7 103.8-8.7 4.7-25.8 11.1-48.4 10.7H2.4c-8.7 50.8 5.8 116.8 44 162.1 37.1 43.9 92.7 66.2 165.4 66.2 157.4 0 273.9-72.5 328.4-204.2 21.4.4 67.6.1 91.3-45.2 1.5-2.5 6.6-13.2 8.5-17.1l-13.3-8.9zm-511.1-27.9h-66v59.4h66.1v-59.4zm78.1 0h-66.1v59.4h66.1v-59.4zm78.1 0h-66.1v59.4h66.1v-59.4zm-78.1-72.1h-66.1v60.1h66.1v-60.1z"/></svg>
</a>
</div>
</div>
</div>
</footer>
</div>
<div class="md-dialog" data-md-component="dialog">
<div class="md-dialog__inner md-typeset"></div>
</div>
<script id="__config" type="application/json">{"base": "../../..", "features": ["content.code.annotate", "content.tabs.link", "navigation.instant", "navigation.sections", "search.highlight", "search.share", "search.suggest", "navigation.tabs", "navigation.tracking", "navigation.tabs.sticky", "navigation.top", "content.code.copy"], "search": "../../../assets/javascripts/workers/search.74e28a9f.min.js", "translations": {"clipboard.copied": "\u5df2\u590d\u5236", "clipboard.copy": "\u590d\u5236", "search.result.more.one": "\u5728\u8be5\u9875\u4e0a\u8fd8\u6709 1 \u4e2a\u7b26\u5408\u6761\u4ef6\u7684\u7ed3\u679c", "search.result.more.other": "\u5728\u8be5\u9875\u4e0a\u8fd8\u6709 # \u4e2a\u7b26\u5408\u6761\u4ef6\u7684\u7ed3\u679c", "search.result.none": "\u6ca1\u6709\u627e\u5230\u7b26\u5408\u6761\u4ef6\u7684\u7ed3\u679c", "search.result.one": "\u627e\u5230 1 \u4e2a\u7b26\u5408\u6761\u4ef6\u7684\u7ed3\u679c", "search.result.other": "# \u4e2a\u7b26\u5408\u6761\u4ef6\u7684\u7ed3\u679c", "search.result.placeholder": "\u952e\u5165\u4ee5\u5f00\u59cb\u641c\u7d22", "search.result.term.missing": "\u7f3a\u5c11", "select.version": "\u9009\u62e9\u5f53\u524d\u7248\u672c"}, "version": {"provider": "mike"}}</script>
<script src="../../../assets/javascripts/bundle.220ee61c.min.js"></script>
</body>
</html>