kafka CompressionRatioEstimator 源码

kafka CompressionRatioEstimator 代码

文件路径：/clients/src/main/java/org/apache/kafka/common/record/CompressionRatioEstimator.java

/*
 * Licensed to the Apache Software Foundation (ASF) under one or more
 * contributor license agreements. See the NOTICE file distributed with
 * this work for additional information regarding copyright ownership.
 * The ASF licenses this file to You under the Apache License, Version 2.0
 * (the "License"); you may not use this file except in compliance with
 * the License. You may obtain a copy of the License at
 *
 *    http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package org.apache.kafka.common.record;

import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ConcurrentMap;


/**
 * This class help estimate the compression ratio for each topic and compression type combination.
 */
public class CompressionRatioEstimator {
    // The constant speed to increase compression ratio when a batch compresses better than expected.
    public static final float COMPRESSION_RATIO_IMPROVING_STEP = 0.005f;
    // The minimum speed to decrease compression ratio when a batch compresses worse than expected.
    public static final float COMPRESSION_RATIO_DETERIORATE_STEP = 0.05f;
    private static final ConcurrentMap<String, float[]> COMPRESSION_RATIO = new ConcurrentHashMap<>();

    /**
     * Update the compression ratio estimation for a topic and compression type.
     *
     * @param topic         the topic to update compression ratio estimation.
     * @param type          the compression type.
     * @param observedRatio the observed compression ratio.
     * @return the compression ratio estimation after the update.
     */
    public static float updateEstimation(String topic, CompressionType type, float observedRatio) {
        float[] compressionRatioForTopic = getAndCreateEstimationIfAbsent(topic);
        float currentEstimation = compressionRatioForTopic[type.id];
        synchronized (compressionRatioForTopic) {
            if (observedRatio > currentEstimation)
                compressionRatioForTopic[type.id] = Math.max(currentEstimation + COMPRESSION_RATIO_DETERIORATE_STEP, observedRatio);
            else if (observedRatio < currentEstimation) {
                compressionRatioForTopic[type.id] = Math.max(currentEstimation - COMPRESSION_RATIO_IMPROVING_STEP, observedRatio);
            }
        }
        return compressionRatioForTopic[type.id];
    }

    /**
     * Get the compression ratio estimation for a topic and compression type.
     */
    public static float estimation(String topic, CompressionType type) {
        float[] compressionRatioForTopic = getAndCreateEstimationIfAbsent(topic);
        return compressionRatioForTopic[type.id];
    }

    /**
     * Reset the compression ratio estimation to the initial values for a topic.
     */
    public static void resetEstimation(String topic) {
        float[] compressionRatioForTopic = getAndCreateEstimationIfAbsent(topic);
        synchronized (compressionRatioForTopic) {
            for (CompressionType type : CompressionType.values()) {
                compressionRatioForTopic[type.id] = type.rate;
            }
        }
    }

    /**
     * Remove the compression ratio estimation for a topic.
     */
    public static void removeEstimation(String topic) {
        COMPRESSION_RATIO.remove(topic);
    }

    /**
     * Set the compression estimation for a topic compression type combination. This method is for unit test purpose.
     */
    public static void setEstimation(String topic, CompressionType type, float ratio) {
        float[] compressionRatioForTopic = getAndCreateEstimationIfAbsent(topic);
        synchronized (compressionRatioForTopic) {
            compressionRatioForTopic[type.id] = ratio;
        }
    }

    private static float[] getAndCreateEstimationIfAbsent(String topic) {
        float[] compressionRatioForTopic = COMPRESSION_RATIO.get(topic);
        if (compressionRatioForTopic == null) {
            compressionRatioForTopic = initialCompressionRatio();
            float[] existingCompressionRatio = COMPRESSION_RATIO.putIfAbsent(topic, compressionRatioForTopic);
            // Someone created the compression ratio array before us, use it.
            if (existingCompressionRatio != null)
                return existingCompressionRatio;
        }
        return compressionRatioForTopic;
    }

    private static float[] initialCompressionRatio() {
        float[] compressionRatio = new float[CompressionType.values().length];
        for (CompressionType type : CompressionType.values()) {
            compressionRatio[type.id] = type.rate;
        }
        return compressionRatio;
    }
}

相关信息

kafka 源码目录

相关文章

kafka AbstractLegacyRecordBatch 源码

kafka AbstractRecordBatch 源码

kafka AbstractRecords 源码

kafka BaseRecords 源码

kafka ByteBufferLogInputStream 源码

kafka CompressionType 源码

kafka ControlRecordType 源码

kafka ControlRecordUtils 源码

kafka ConvertedRecords 源码

kafka DefaultRecord 源码