001/* 002 * Licensed to the Apache Software Foundation (ASF) under one or more 003 * contributor license agreements. See the NOTICE file distributed with 004 * this work for additional information regarding copyright ownership. 005 * The ASF licenses this file to You under the Apache license, Version 2.0 006 * (the "License"); you may not use this file except in compliance with 007 * the License. You may obtain a copy of the License at 008 * 009 * http://www.apache.org/licenses/LICENSE-2.0 010 * 011 * Unless required by applicable law or agreed to in writing, software 012 * distributed under the License is distributed on an "AS IS" BASIS, 013 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 014 * See the license for the specific language governing permissions and 015 * limitations under the license. 016 */ 017package org.apache.logging.log4j.flume.appender; 018 019import java.io.ByteArrayOutputStream; 020import java.io.IOException; 021import java.util.ArrayList; 022import java.util.HashMap; 023import java.util.List; 024import java.util.Map; 025import java.util.zip.GZIPOutputStream; 026 027import org.apache.flume.event.SimpleEvent; 028import org.apache.logging.log4j.Level; 029import org.apache.logging.log4j.LoggingException; 030import org.apache.logging.log4j.Marker; 031import org.apache.logging.log4j.ThreadContext; 032import org.apache.logging.log4j.core.LogEvent; 033import org.apache.logging.log4j.core.impl.Log4jLogEvent; 034import org.apache.logging.log4j.core.impl.ThrowableProxy; 035import org.apache.logging.log4j.core.time.Instant; 036import org.apache.logging.log4j.core.util.Patterns; 037import org.apache.logging.log4j.core.util.UuidUtil; 038import org.apache.logging.log4j.message.MapMessage; 039import org.apache.logging.log4j.message.Message; 040import org.apache.logging.log4j.message.StructuredDataId; 041import org.apache.logging.log4j.message.StructuredDataMessage; 042import org.apache.logging.log4j.util.ReadOnlyStringMap; 043import org.apache.logging.log4j.util.Strings; 044 045/** 046 * Class that is both a Flume and Log4j Event. 047 */ 048public class FlumeEvent extends SimpleEvent implements LogEvent { 049 050 static final String GUID = "guId"; 051 /** 052 * Generated serial version ID. 053 */ 054 private static final long serialVersionUID = -8988674608627854140L; 055 056 private static final String DEFAULT_MDC_PREFIX = Strings.EMPTY; 057 058 private static final String DEFAULT_EVENT_PREFIX = Strings.EMPTY; 059 060 private static final String EVENT_TYPE = "eventType"; 061 062 private static final String EVENT_ID = "eventId"; 063 064 private static final String TIMESTAMP = "timeStamp"; 065 066 private final LogEvent event; 067 068 private final Map<String, String> contextMap = new HashMap<>(); 069 070 private final boolean compress; 071 072 /** 073 * Construct the FlumeEvent. 074 * @param event The Log4j LogEvent. 075 * @param includes A comma separated list of MDC elements to include. 076 * @param excludes A comma separated list of MDC elements to exclude. 077 * @param required A comma separated list of MDC elements that are required to be defined. 078 * @param mdcPrefix The value to prefix to MDC keys. 079 * @param eventPrefix The value to prefix to event keys. 080 * @param compress If true the event body should be compressed. 081 */ 082 public FlumeEvent(final LogEvent event, final String includes, final String excludes, final String required, 083 String mdcPrefix, String eventPrefix, final boolean compress) { 084 this.event = event; 085 this.compress = compress; 086 final Map<String, String> headers = getHeaders(); 087 headers.put(TIMESTAMP, Long.toString(event.getTimeMillis())); 088 if (mdcPrefix == null) { 089 mdcPrefix = DEFAULT_MDC_PREFIX; 090 } 091 if (eventPrefix == null) { 092 eventPrefix = DEFAULT_EVENT_PREFIX; 093 } 094 final Map<String, String> mdc = event.getContextData().toMap(); 095 if (includes != null) { 096 final String[] array = includes.split(Patterns.COMMA_SEPARATOR); 097 if (array.length > 0) { 098 for (String str : array) { 099 str = str.trim(); 100 if (mdc.containsKey(str)) { 101 contextMap.put(str, mdc.get(str)); 102 } 103 } 104 } 105 } else if (excludes != null) { 106 final String[] array = excludes.split(Patterns.COMMA_SEPARATOR); 107 if (array.length > 0) { 108 final List<String> list = new ArrayList<>(array.length); 109 for (final String value : array) { 110 list.add(value.trim()); 111 } 112 for (final Map.Entry<String, String> entry : mdc.entrySet()) { 113 if (!list.contains(entry.getKey())) { 114 contextMap.put(entry.getKey(), entry.getValue()); 115 } 116 } 117 } 118 } else { 119 contextMap.putAll(mdc); 120 } 121 122 if (required != null) { 123 final String[] array = required.split(Patterns.COMMA_SEPARATOR); 124 if (array.length > 0) { 125 for (String str : array) { 126 str = str.trim(); 127 if (!mdc.containsKey(str)) { 128 throw new LoggingException("Required key " + str + " is missing from the MDC"); 129 } 130 } 131 } 132 } 133 final String guid = UuidUtil.getTimeBasedUuid().toString(); 134 final Message message = event.getMessage(); 135 if (message instanceof MapMessage) { 136 // Add the guid to the Map so that it can be included in the Layout. 137 @SuppressWarnings("unchecked") 138 final 139 MapMessage<?, String> stringMapMessage = (MapMessage<?, String>) message; 140 stringMapMessage.put(GUID, guid); 141 if (message instanceof StructuredDataMessage) { 142 addStructuredData(eventPrefix, headers, (StructuredDataMessage) message); 143 } 144 addMapData(eventPrefix, headers, stringMapMessage); 145 } else { 146 headers.put(GUID, guid); 147 } 148 149 addContextData(mdcPrefix, headers, contextMap); 150 } 151 152 protected void addStructuredData(final String prefix, final Map<String, String> fields, 153 final StructuredDataMessage msg) { 154 fields.put(prefix + EVENT_TYPE, msg.getType()); 155 final StructuredDataId id = msg.getId(); 156 fields.put(prefix + EVENT_ID, id.getName()); 157 } 158 159 protected void addMapData(final String prefix, final Map<String, String> fields, final MapMessage<?, String> msg) { 160 final Map<String, String> data = msg.getData(); 161 for (final Map.Entry<String, String> entry : data.entrySet()) { 162 fields.put(prefix + entry.getKey(), entry.getValue()); 163 } 164 } 165 166 protected void addContextData(final String prefix, final Map<String, String> fields, 167 final Map<String, String> context) { 168 final Map<String, String> map = new HashMap<>(); 169 for (final Map.Entry<String, String> entry : context.entrySet()) { 170 if (entry.getKey() != null && entry.getValue() != null) { 171 fields.put(prefix + entry.getKey(), entry.getValue()); 172 map.put(prefix + entry.getKey(), entry.getValue()); 173 } 174 } 175 context.clear(); 176 context.putAll(map); 177 } 178 179 @Override 180 public LogEvent toImmutable() { 181 return Log4jLogEvent.createMemento(this); 182 } 183 184 /** 185 * Set the body in the event. 186 * @param body The body to add to the event. 187 */ 188 @Override 189 public void setBody(final byte[] body) { 190 if (body == null || body.length == 0) { 191 super.setBody(new byte[0]); 192 return; 193 } 194 if (compress) { 195 final ByteArrayOutputStream baos = new ByteArrayOutputStream(); 196 try (GZIPOutputStream os = new GZIPOutputStream(baos)) { 197 os.write(body); 198 } catch (final IOException ioe) { 199 throw new LoggingException("Unable to compress message", ioe); 200 } 201 super.setBody(baos.toByteArray()); 202 } else { 203 super.setBody(body); 204 } 205 } 206 207 /** 208 * Get the Frequently Qualified Class Name. 209 * @return the FQCN String. 210 */ 211 @Override 212 public String getLoggerFqcn() { 213 return event.getLoggerFqcn(); 214 } 215 216 /** 217 * Returns the logging Level. 218 * @return the Level. 219 */ 220 @Override 221 public Level getLevel() { 222 return event.getLevel(); 223 } 224 225 /** 226 * Returns the logger name. 227 * @return the logger name. 228 */ 229 @Override 230 public String getLoggerName() { 231 return event.getLoggerName(); 232 } 233 234 /** 235 * Returns the StackTraceElement for the caller of the logging API. 236 * @return the StackTraceElement of the caller. 237 */ 238 @Override 239 public StackTraceElement getSource() { 240 return event.getSource(); 241 } 242 243 /** 244 * Returns the Message. 245 * @return the Message. 246 */ 247 @Override 248 public Message getMessage() { 249 return event.getMessage(); 250 } 251 252 /** 253 * Returns the Marker. 254 * @return the Marker. 255 */ 256 @Override 257 public Marker getMarker() { 258 return event.getMarker(); 259 } 260 261 /** 262 * Returns the ID of the Thread. 263 * @return the ID of the Thread. 264 */ 265 @Override 266 public long getThreadId() { 267 return event.getThreadId(); 268 } 269 270 /** 271 * Returns the priority of the Thread. 272 * @return the priority of the Thread. 273 */ 274 @Override 275 public int getThreadPriority() { 276 return event.getThreadPriority(); 277 } 278 279 /** 280 * Returns the name of the Thread. 281 * @return the name of the Thread. 282 */ 283 @Override 284 public String getThreadName() { 285 return event.getThreadName(); 286 } 287 288 /** 289 * Returns the event timestamp. 290 * @return the event timestamp. 291 */ 292 @Override 293 public long getTimeMillis() { 294 return event.getTimeMillis(); 295 } 296 297 /** 298 * {@inheritDoc} 299 * @since 2.11 300 */ 301 @Override 302 public Instant getInstant() { 303 return event.getInstant(); 304 } 305 306 /** 307 * Returns the value of the running Java Virtual Machine's high-resolution time source when this event was created, 308 * or a dummy value if it is known that this value will not be used downstream. 309 * @return the event nanosecond timestamp. 310 */ 311 @Override 312 public long getNanoTime() { 313 return event.getNanoTime(); 314 } 315 316 /** 317 * Returns the Throwable associated with the event, if any. 318 * @return the Throwable. 319 */ 320 @Override 321 public Throwable getThrown() { 322 return event.getThrown(); 323 } 324 325 /** 326 * Returns the Throwable associated with the event, if any. 327 * @return the Throwable. 328 */ 329 @Override 330 public ThrowableProxy getThrownProxy() { 331 return event.getThrownProxy(); 332 } 333 334 /** 335 * Returns a copy of the context Map. 336 * @return a copy of the context Map. 337 */ 338 @Override 339 public Map<String, String> getContextMap() { 340 return contextMap; 341 } 342 343 /** 344 * Returns the context data of the {@code LogEvent} that this {@code FlumeEvent} was constructed with. 345 * @return the context data of the {@code LogEvent} that this {@code FlumeEvent} was constructed with. 346 */ 347 @Override 348 public ReadOnlyStringMap getContextData() { 349 return event.getContextData(); 350 } 351 352 /** 353 * Returns a copy of the context stack. 354 * @return a copy of the context stack. 355 */ 356 @Override 357 public ThreadContext.ContextStack getContextStack() { 358 return event.getContextStack(); 359 } 360 361 @Override 362 public boolean isIncludeLocation() { 363 return event.isIncludeLocation(); 364 } 365 366 @Override 367 public void setIncludeLocation(final boolean includeLocation) { 368 event.setIncludeLocation(includeLocation); 369 } 370 371 @Override 372 public boolean isEndOfBatch() { 373 return event.isEndOfBatch(); 374 } 375 376 @Override 377 public void setEndOfBatch(final boolean endOfBatch) { 378 event.setEndOfBatch(endOfBatch); 379 } 380 381}