|
# Licensed to the Apache Software Foundation (ASF) under one
|
||
|
|
# or more contributor license agreements. See the NOTICE file
|
||
|
|
# distributed with this work for additional information
|
||
|
|
# regarding copyright ownership. The ASF licenses this file
|
||
|
|
# to you under the Apache License, Version 2.0 (the
|
||
|
|
# "License"); you may not use this file except in compliance
|
||
|
|
# with the License. You may obtain a copy of the License at
|
||
|
|
#
|
||
|
|
# http://www.apache.org/licenses/LICENSE-2.0
|
||
|
|
#
|
||
|
|
# Unless required by applicable law or agreed to in writing,
|
||
|
|
# software distributed under the License is distributed on an
|
||
|
|
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||
|
|
# KIND, either express or implied. See the License for the
|
||
|
|
# specific language governing permissions and limitations
|
||
|
|
# under the License.
|
||
|
|
|
||
|
|
import os
|
||
|
|
import random
|
||
|
|
|
||
|
|
def estimate_density(DATA_PATH, feature_size):
|
||
|
|
"""sample 10 times of a size of 1000 for estimating the density of the sparse dataset"""
|
||
|
|
if not os.path.exists(DATA_PATH):
|
||
|
|
raise Exception("Data is not there!")
|
||
|
|
density = []
|
||
|
|
P = 0.01
|
||
|
|
for _ in xrange(10):
|
||
|
|
num_non_zero = 0
|
||
|
|
num_sample = 0
|
||
|
|
with open(DATA_PATH) as f:
|
||
|
|
for line in f:
|
||
|
|
if (random.random() < P):
|
||
|
|
num_non_zero += len(line.split(" ")) - 1
|
||
|
|
num_sample += 1
|
||
|
|
density.append(num_non_zero * 1.0 / (feature_size * num_sample))
|
||
|
|
return sum(density) / len(density)
|
||
|
|
|