001 /*
002 * Licensed to the Apache Software Foundation (ASF) under one or more
003 * contributor license agreements. See the NOTICE file distributed with
004 * this work for additional information regarding copyright ownership.
005 * The ASF licenses this file to You under the Apache license, Version 2.0
006 * (the "License"); you may not use this file except in compliance with
007 * the License. You may obtain a copy of the License at
008 *
009 * http://www.apache.org/licenses/LICENSE-2.0
010 *
011 * Unless required by applicable law or agreed to in writing, software
012 * distributed under the License is distributed on an "AS IS" BASIS,
013 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
014 * See the license for the specific language governing permissions and
015 * limitations under the license.
016 */
017 package org.apache.logging.log4j.core.appender;
018
019 import java.io.File;
020 import java.io.IOException;
021 import java.io.OutputStream;
022 import java.io.RandomAccessFile;
023 import java.io.Serializable;
024 import java.lang.reflect.Method;
025 import java.nio.ByteOrder;
026 import java.nio.MappedByteBuffer;
027 import java.nio.channels.FileChannel;
028 import java.security.AccessController;
029 import java.security.PrivilegedActionException;
030 import java.security.PrivilegedExceptionAction;
031 import java.util.HashMap;
032 import java.util.Map;
033
034 import org.apache.logging.log4j.core.Layout;
035 import org.apache.logging.log4j.core.util.Assert;
036 import org.apache.logging.log4j.core.util.Closer;
037 import org.apache.logging.log4j.core.util.NullOutputStream;
038
039 /**
040 * Extends OutputStreamManager but instead of using a buffered output stream, this class maps a region of a file into
041 * memory and writes to this memory region.
042 * <p>
043 *
044 * @see <a
045 * href="http://www.codeproject.com/Tips/683614/Things-to-Know-about-Memory-Mapped-File-in-Java">http://www.codeproject.com/Tips/683614/Things-to-Know-about-Memory-Mapped-File-in-Java</a>
046 * @see <a href="http://bugs.java.com/view_bug.do?bug_id=6893654">http://bugs.java.com/view_bug.do?bug_id=6893654</a>
047 * @see <a href="http://bugs.java.com/view_bug.do?bug_id=4724038">http://bugs.java.com/view_bug.do?bug_id=4724038</a>
048 * @see <a
049 * href="http://stackoverflow.com/questions/9261316/memory-mapped-mappedbytebuffer-or-direct-bytebuffer-for-db-implementation">http://stackoverflow.com/questions/9261316/memory-mapped-mappedbytebuffer-or-direct-bytebuffer-for-db-implementation</a>
050 *
051 * @since 2.1
052 */
053 public class MemoryMappedFileManager extends OutputStreamManager {
054 static final int DEFAULT_REGION_LENGTH = 32 * 1024 * 1024;
055 private static final MemoryMappedFileManagerFactory FACTORY = new MemoryMappedFileManagerFactory();
056
057 private final boolean isForce;
058 private final int regionLength;
059 private final String advertiseURI;
060 private final RandomAccessFile randomAccessFile;
061 private final ThreadLocal<Boolean> isEndOfBatch = new ThreadLocal<Boolean>();
062 private MappedByteBuffer mappedBuffer;
063 private long mappingOffset;
064
065 protected MemoryMappedFileManager(final RandomAccessFile file, final String fileName, final OutputStream os,
066 final boolean force, final long position, final int regionLength, final String advertiseURI,
067 final Layout<? extends Serializable> layout) throws IOException {
068 super(os, fileName, layout);
069 this.isForce = force;
070 this.randomAccessFile = Assert.requireNonNull(file, "RandomAccessFile");
071 this.regionLength = regionLength;
072 this.advertiseURI = advertiseURI;
073 this.isEndOfBatch.set(Boolean.FALSE);
074 this.mappedBuffer = mmap(randomAccessFile.getChannel(), getFileName(), position, regionLength);
075 this.mappingOffset = position;
076 }
077
078 /**
079 * Returns the MemoryMappedFileManager.
080 *
081 * @param fileName The name of the file to manage.
082 * @param append true if the file should be appended to, false if it should be overwritten.
083 * @param isForce true if the contents should be flushed to disk on every write
084 * @param regionLength The mapped region length.
085 * @param advertiseURI the URI to use when advertising the file
086 * @param layout The layout.
087 * @return A MemoryMappedFileManager for the File.
088 */
089 public static MemoryMappedFileManager getFileManager(final String fileName, final boolean append,
090 final boolean isForce, final int regionLength, final String advertiseURI,
091 final Layout<? extends Serializable> layout) {
092 return (MemoryMappedFileManager) getManager(fileName, new FactoryData(append, isForce, regionLength,
093 advertiseURI, layout), FACTORY);
094 }
095
096 public Boolean isEndOfBatch() {
097 return isEndOfBatch.get();
098 }
099
100 public void setEndOfBatch(final boolean isEndOfBatch) {
101 this.isEndOfBatch.set(Boolean.valueOf(isEndOfBatch));
102 }
103
104 @Override
105 protected synchronized void write(final byte[] bytes, int offset, int length) {
106 super.write(bytes, offset, length); // writes to dummy output stream
107
108 while (length > mappedBuffer.remaining()) {
109 final int chunk = mappedBuffer.remaining();
110 mappedBuffer.put(bytes, offset, chunk);
111 offset += chunk;
112 length -= chunk;
113 remap();
114 }
115 mappedBuffer.put(bytes, offset, length);
116
117 // no need to call flush() if force is true,
118 // already done in AbstractOutputStreamAppender.append
119 }
120
121 private synchronized void remap() {
122 final long offset = this.mappingOffset + mappedBuffer.position();
123 final int length = mappedBuffer.remaining() + regionLength;
124 try {
125 unsafeUnmap(mappedBuffer);
126 final long fileLength = randomAccessFile.length() + regionLength;
127 LOGGER.debug("MMapAppender extending {} by {} bytes to {}", getFileName(), regionLength, fileLength);
128
129 long startNanos = System.nanoTime();
130 randomAccessFile.setLength(fileLength);
131 final float millis = (float) ((System.nanoTime() - startNanos) / (1000.0 * 1000.0));
132 LOGGER.debug("MMapAppender extended {} OK in {} millis", getFileName(), millis);
133
134 mappedBuffer = mmap(randomAccessFile.getChannel(), getFileName(), offset, length);
135 mappingOffset = offset;
136 } catch (final Exception ex) {
137 LOGGER.error("Unable to remap " + getName() + ". " + ex);
138 }
139 }
140
141 @Override
142 public synchronized void flush() {
143 mappedBuffer.force();
144 }
145
146 @Override
147 public synchronized void close() {
148 final long position = mappedBuffer.position();
149 final long length = mappingOffset + position;
150 try {
151 unsafeUnmap(mappedBuffer);
152 } catch (final Exception ex) {
153 LOGGER.error("Unable to unmap MappedBuffer " + getName() + ". " + ex);
154 }
155 try {
156 LOGGER.debug("MMapAppender closing. Setting {} length to {} (offset {} + position {})", getFileName(),
157 length, mappingOffset, position);
158 randomAccessFile.setLength(length);
159 randomAccessFile.close();
160 } catch (final IOException ex) {
161 LOGGER.error("Unable to close MemoryMappedFile " + getName() + ". " + ex);
162 }
163 }
164
165 public static MappedByteBuffer mmap(final FileChannel fileChannel, final String fileName, final long start,
166 final int size) throws IOException {
167 for (int i = 1;; i++) {
168 try {
169 LOGGER.debug("MMapAppender remapping {} start={}, size={}", fileName, start, size);
170
171 final long startNanos = System.nanoTime();
172 final MappedByteBuffer map = fileChannel.map(FileChannel.MapMode.READ_WRITE, start, size);
173 map.order(ByteOrder.nativeOrder());
174
175 final float millis = (float) ((System.nanoTime() - startNanos) / (1000.0 * 1000.0));
176 LOGGER.debug("MMapAppender remapped {} OK in {} millis", fileName, millis);
177
178 return map;
179 } catch (final IOException e) {
180 if (e.getMessage() == null || !e.getMessage().endsWith("user-mapped section open")) {
181 throw e;
182 }
183 LOGGER.debug("Remap attempt {}/10 failed. Retrying...", i, e);
184 if (i < 10) {
185 Thread.yield();
186 } else {
187 try {
188 Thread.sleep(1);
189 } catch (final InterruptedException ignored) {
190 Thread.currentThread().interrupt();
191 throw e;
192 }
193 }
194 }
195 }
196 }
197
198 private static void unsafeUnmap(final MappedByteBuffer mbb) throws PrivilegedActionException {
199 LOGGER.debug("MMapAppender unmapping old buffer...");
200 final long startNanos = System.nanoTime();
201 AccessController.doPrivileged(new PrivilegedExceptionAction<Object>() {
202 @Override
203 public Object run() throws Exception {
204 final Method getCleanerMethod = mbb.getClass().getMethod("cleaner");
205 getCleanerMethod.setAccessible(true);
206 final Object cleaner = getCleanerMethod.invoke(mbb); // sun.misc.Cleaner instance
207 final Method cleanMethod = cleaner.getClass().getMethod("clean");
208 cleanMethod.invoke(cleaner);
209 return null;
210 }
211 });
212 final float millis = (float) ((System.nanoTime() - startNanos) / (1000.0 * 1000.0));
213 LOGGER.debug("MMapAppender unmapped buffer OK in {} millis", millis);
214 }
215
216 /**
217 * Returns the name of the File being managed.
218 *
219 * @return The name of the File being managed.
220 */
221 public String getFileName() {
222 return getName();
223 }
224
225 /**
226 * Returns the length of the memory mapped region.
227 *
228 * @return the length of the mapped region
229 */
230 public int getRegionLength() {
231 return regionLength;
232 }
233
234 /**
235 * Returns {@code true} if the content of the buffer should be forced to the storage device on every write,
236 * {@code false} otherwise.
237 *
238 * @return whether each write should be force-sync'ed
239 */
240 public boolean isImmediateFlush() {
241 return isForce;
242 }
243
244 /**
245 * Gets this FileManager's content format specified by:
246 * <p>
247 * Key: "fileURI" Value: provided "advertiseURI" param.
248 * </p>
249 *
250 * @return Map of content format keys supporting FileManager
251 */
252 @Override
253 public Map<String, String> getContentFormat() {
254 final Map<String, String> result = new HashMap<String, String>(super.getContentFormat());
255 result.put("fileURI", advertiseURI);
256 return result;
257 }
258
259 /**
260 * Factory Data.
261 */
262 private static class FactoryData {
263 private final boolean append;
264 private final boolean force;
265 private final int regionLength;
266 private final String advertiseURI;
267 private final Layout<? extends Serializable> layout;
268
269 /**
270 * Constructor.
271 *
272 * @param append Append to existing file or truncate.
273 * @param force forces the memory content to be written to the storage device on every event
274 * @param regionLength length of the mapped region
275 */
276 public FactoryData(final boolean append, final boolean force, final int regionLength,
277 final String advertiseURI, final Layout<? extends Serializable> layout) {
278 this.append = append;
279 this.force = force;
280 this.regionLength = regionLength;
281 this.advertiseURI = advertiseURI;
282 this.layout = layout;
283 }
284 }
285
286 /**
287 * Factory to create a MemoryMappedFileManager.
288 */
289 private static class MemoryMappedFileManagerFactory implements ManagerFactory<MemoryMappedFileManager, FactoryData> {
290
291 /**
292 * Create a MemoryMappedFileManager.
293 *
294 * @param name The name of the File.
295 * @param data The FactoryData
296 * @return The MemoryMappedFileManager for the File.
297 */
298 @SuppressWarnings("resource")
299 @Override
300 public MemoryMappedFileManager createManager(final String name, final FactoryData data) {
301 final File file = new File(name);
302 final File parent = file.getParentFile();
303 if (null != parent && !parent.exists()) {
304 parent.mkdirs();
305 }
306 if (!data.append) {
307 file.delete();
308 }
309
310 final OutputStream os = NullOutputStream.NULL_OUTPUT_STREAM;
311 RandomAccessFile raf = null;
312 try {
313 raf = new RandomAccessFile(name, "rw");
314 final long position = (data.append) ? raf.length() : 0;
315 raf.setLength(position + data.regionLength);
316 return new MemoryMappedFileManager(raf, name, os, data.force, position, data.regionLength,
317 data.advertiseURI, data.layout);
318 } catch (final Exception ex) {
319 LOGGER.error("MemoryMappedFileManager (" + name + ") " + ex);
320 Closer.closeSilently(raf);
321 }
322 return null;
323 }
324 }
325 }