Contextual Bandit算法在推荐系统中的实现及应用

public int consume(Event newEvent, Config config) {
    if (newEvent.getPid() != pid)
        return -1;

    reset(config);

    int showConsumedCount = 0;
    int clickConsumedCount = 0;
    long current = System.currentTimeMillis();
    Iterator<Event> iter = queue.iterator();
    while (iter.hasNext()) {
        Event event = iter.next();
        if (event.isSameFlow(newEvent)) {
            if (newEvent.isShow()) {
                newEvent = null; // 重复曝光，或者点击事件在曝光事件前到达
                break;
            }
            if (newEvent.isClick() && event.isClick()) {
                newEvent = null; // 重复点击事件
                break;
            }
            iter.remove(); // 高优先级的事情会覆盖低优先级的事情
            if (null == newEvent.getItemTrackData()) {
                newEvent.setItemTrackData(event.getItemTrackData());
            }
            continue;
        }
        if (current - event.getTimestamp() < config.cacheTimeSpan)
            continue;

        INDArray xt = event.getFeature(config);
        INDArray x = xt.transpose();
        A.addi(x.mmul(xt));
        double reward = event.getReward();
        if (reward > 0.0)
            b.addi(x.muli(reward));
        iter.remove();
        if (event.isShow())
            showConsumedCount ++;
        else if (event.isClick())
            clickConsumedCount ++;
    }
    if (null == newEvent)
        return 0;
    if (newEvent.getType().equals(EventType.BUY)) {
        // 购买事情立即消费
        INDArray xt = newEvent.getFeature(config);
        if (null == xt) {
            LogPushUtil.push("BuyFeatureMissingEvent");
        }
        else if (xt.length() == b.length()) {
            INDArray x = xt.transpose();
            A.addi(x.mmul(xt));
            double reward = newEvent.getReward();
            b.addi(x.muli(reward));
            LogPushUtil.push("BuyEventConsumed");
            records += config.updateRecords;
        }
    } else if (StringUtils.isNotEmpty(newEvent.getItemTrackData())) {
        queue.add(newEvent);
        checkQueueIsFullOrNot(config.cacheTimeSpan / 2, config);
    }
    if (showConsumedCount > 0) {
        records += showConsumedCount;
        LogPushUtil.push("ShowEventConsumed", showConsumedCount);
    }
    if (clickConsumedCount > 0) {
        records += clickConsumedCount;
        LogPushUtil.push("ClickEventConsumed", clickConsumedCount);
    }
    if (records >= config.updateRecords) {
        writeToRedis(config);
    }
    return records;
}

private void parallelGetLinUcbScore(Session session, INDArray features, List<Item> items) throws InterruptedException {
    int concurrency = Math.min(session.config.linUcbConcurrency, items.size());
    int num = items.size() / concurrency;

    ExecutorService es = session.context.getThreadExecutorService(session.reqId, session.reqId.getSceneId());
    final CountDownLatch countDownLatch = new CountDownLatch(concurrency);
    int left = items.size() - num * concurrency;
    int start = 0, end = num;
    for (int i = 0; i < concurrency; ++i) {
        if (i < left)
            end++;
        Logger.debug("prepare to start linucb score sub thread: [" + start + ", " + end + "]");
        int finalStart = start;
        int finalEnd = end;
        Runnable runnable = () -> {
            long startTime = System.currentTimeMillis();
            try {
                getLinUcbScore(session, features, items, finalStart, finalEnd);
            } catch (Exception e) {
                String exception = ExceptionUtils.getFullStackTrace(e);
                Logger.error("linucb worker exception:", exception);
            }
            finally {
                countDownLatch.countDown();
            }
            long time = System.currentTimeMillis() - startTime;
            Logger.info("[Timer] compute sub linUcb scores [", finalStart, ", ", finalEnd, "] taken", time, "ms");
        };
        es.execute(TtlRunnable.get(runnable));
        start = end;
        end = start + num;
    }
    countDownLatch.await(session.config.linUcbTimeOut, TimeUnit.MILLISECONDS);
}

private void getLinUcbScore(Session session, INDArray features, List<Item> items, int start, int end) throws IOException {
    if (items.isEmpty() || null == features)
        return;
    StopWatch stopWatch = StopWatch.CreateStopWatchAndStart();
    for (int i = start; i < end; ++i) {
        stopWatch.restart();
        Item item = items.get(i);
        long pid = item.getProductId();
        byte[] matrix = getLinUcbMatrix(session, pid);

        INDArray feature = features.getRow(i);
        INDArray featureT = feature.transpose();
        Logger.detail("<", i, ">", pid, "item feature:", item.getTrackInfo(), "vector:", feature);
        if (matrix == null) {
            double p = session.config.linUcbAlpha * FastMath.sqrt(feature.mmul(featureT).getDouble(0));
            item.setMatchScore(p);
            Logger.info("<", i, "> linucb new item:", pid, "ucb score:", p);
            continue;
        }

        ByteArrayInputStream in = new ByteArrayInputStream(matrix);
        INDArray[] Ab = ByteSerde.fromByteArrayStream(in);
        if (Ab.length < 2)
            continue;
        INDArray invertA = Ab[0];
        INDArray b = Ab[1];
        Logger.detail("<", i, ">", pid, "[Timer] fetch matrix taken", stopWatch.click(), "ms");
        if (b.length() != feature.length()) {
            Logger.error("the length of b is", b.length(), "should be", feature.length());
            double p = session.config.linUcbAlpha * FastMath.sqrt(feature.mmuli(featureT).getFloat(0));
            item.setMatchScore(p);
            continue;
        }
        INDArray theta = invertA.mmul(b);
        INDArray thetaX = theta.transposei().mmul(featureT);
        INDArray temp = feature.mmuli(invertA).mmul(featureT);
        double p = thetaX.getDouble(0);
        item.setPreferScore(p);
        p += session.config.linUcbAlpha * FastMath.sqrt(temp.getDouble(0));
        item.setMatchScore(p);
        Logger.detail("<", i, ">", pid, "[Timer] one iteration linucb score taken", stopWatch.click(), "ms. match score:", p);
    }
}