Custom callbacks in keras for validation dataset testing

Setting option not working in Windows 10 (Problem solved)

 To do this, open the PowerShell (just type “powershell” into the Start menu Search, then right-click it and “Run as administrator”) and enter the following command:

Get-AppXPackage | Foreach {Add-AppxPackage -DisableDevelopmentMode -Register "$($_.InstallLocation)\AppXManifest.xml"}

That will re-register and reinstall all Windows apps, hopefully getting the Settings app (and others) back to full working order.

https://www.maketecheasier.com/fix-settings-app-not-working-in-windows-10/

Chess FEN to bitboard conversion in python

import chess
import numpy as np
board = chess.Board('rnbqkbnr/pppppppp/8/8/8/8/PPPPPPPP/RNBQKBNR w KQkq - 0 1')

w_pawn = (np.asarray(board.pieces(chess.PAWN, chess.WHITE).tolist())).astype(int)
w_rook = (np.asarray(board.pieces(chess.ROOK, chess.WHITE).tolist())).astype(int)
w_knight = (np.asarray(board.pieces(chess.KNIGHT, chess.WHITE).tolist())).astype(int)
w_bishop = (np.asarray(board.pieces(chess.BISHOP, chess.WHITE).tolist())).astype(int)
w_queen = (np.asarray(board.pieces(chess.QUEEN, chess.WHITE).tolist())).astype(int)
w_king = (np.asarray(board.pieces(chess.KING, chess.WHITE).tolist())).astype(int)

b_pawn = (np.asarray(board.pieces(chess.PAWN, chess.BLACK).tolist())).astype(int)
b_rook = (np.asarray(board.pieces(chess.ROOK, chess.BLACK).tolist())).astype(int)
b_knight = (np.asarray(board.pieces(chess.KNIGHT, chess.BLACK).tolist())).astype(int)
b_bishop = (np.asarray(board.pieces(chess.BISHOP, chess.BLACK).tolist())).astype(int)
b_queen = (np.asarray(board.pieces(chess.QUEEN, chess.BLACK).tolist())).astype(int)
b_king = (np.asarray(board.pieces(chess.KING, chess.BLACK).tolist())).astype(int)

print( np.concatenate((w_pawn, w_rook, w_knight, w_bishop, w_queen, w_king,
                       b_pawn, b_rook, b_knight, b_bishop, b_queen, b_king)))


original link

MSE vs Cross Entropy

MSE happens when you assume the error follows Normal Distribution and cross-entropy when you assume binomial distribution.

Disaster detection

https://www.pyimagesearch.com/2019/11/11/detecting-natural-disasters-with-keras-and-deep-learning/?utm_source=facebook&utm_medium=ad-11-11-2019&utm_campaign=11+November+2019+BP+-+Traffic&utm_content=Default+name+-+Traffic&fbid_campaign=6128556144046&fbid_adset=6128556297446&utm_adset=11+November+2019+BP+-+Email+List+-+Worldwide+-+18%2B&fbid_ad=6128556297646

python receive text and predict using trained model

#!/usr/bin/env python
from keras.models import load_model
from keras.layers.core import Reshape, Flatten
from keras.callbacks import ModelCheckpoint
#from data_helpers import load_data
from keras.optimizers import Adam
from keras.models import Model
from sklearn.metrics import confusion_matrix, precision_score, recall_score, f1_score, classification_report
from keras.layers.merge import Concatenate
from sklearn.model_selection import train_test_split
import numpy as np
import re
import sys
def clean_str(string):
    """
    Tokenization/string cleaning for datasets.
    Original taken from https://github.com/yoonkim/CNN_sentence/blob/master/process_data.py
    """
    string = re.sub(r"[^A-Za-z0-9(),!?\'\`]", " ", string)
    string = re.sub(r"\'s", " \'s", string)
    string = re.sub(r"\'ve", " \'ve", string)
    string = re.sub(r"n\'t", " n\'t", string)
    string = re.sub(r"\'re", " \'re", string)
    string = re.sub(r"\'d", " \'d", string)
    string = re.sub(r"\'ll", " \'ll", string)
    string = re.sub(r",", " , ", string)
    string = re.sub(r"!", " ! ", string)
    string = re.sub(r"\(", " \( ", string)
    string = re.sub(r"\)", " \) ", string)
    string = re.sub(r"\?", " \? ", string)
    string = re.sub(r"\s{2,}", " ", string)
    return string.strip().lower()
def pad_sentences(sentences, padding_word="<PAD/>"):
    """
    Pads all sentences to the same length. The length is defined by the longest sentence.
    Returns padded sentences.
    """
    sequence_length = max(len(x) for x in sentences)
    padded_sentences = []
    #print len(sentences)
    for i in range(len(sentences)):
        sentence = sentences[i]
        num_padding = 85 - len(sentence)
        new_sentence = sentence + [padding_word] * num_padding
        padded_sentences.append(new_sentence)
    return padded_sentences

#print ('Loading data')
#x, y, vocabulary, vocabulary_inv = load_data()
#x="@MoriTaheripour shut up nigger whore! Hope u get raped by one of those animals. Might change your tune."
x = sys.argv[1]
#print x
x_text = [clean_str(x)]
#print (x_text)
x_text = [s.split(" ") for s in x_text]
#print x_text
sentences_padded = pad_sentences(x_text)
#print (sentences_padded)
vocabulary = np.load('data123-vocab-servertest2.npy').item()
#for word in sentences_padded:
#   for word2 in word:
#       print vocabulary[word2]

x2 = np.array([[vocabulary[word2] for word2 in word]for word in sentences_padded] )
#print x2
#X_train, X_test, y_train, y_test = train_test_split( x, y, test_size=0.2, random_state=42)
#print type(vocabulary)
#np.save('data1-vocab.npy', vocabulary)
#sequence_length = x.shape[1]
#vocabulary_size = len(vocabulary_inv)
#embedding_dim = 256
#filter_sizes = [3,4,5]
#num_filters = 512
#drop = 0.5

#nb_epoch = 10
#batch_size = 30
a
model = load_model('cnn2D-data123-multi-servertest2.hdf5')
#print(X_test)
y_pred = model.predict(x2)
cc=['Hate','Offensive','Neutral']

#f = open("outputfile.csv","w+")
for xx in y_pred:
    count=0
    for yy in xx:
        print (cc[count] +" " + str(format(yy*100,'.2f')) + "%")
        #f.write(cc[count] +", " + str(format(yy*100,'.2f')) + "%")
        count = count + 1
#f.close()


#y_pred = y_pred.round()
#y_pred = y_pred.astype('int')
#print(y_pred)
#print(y_test)
#y_pred1 = model.predict(X_test).argmax(axis=1)
#y_test1 = y_test.argmax(axis =1)
#print(y_pred1)
#print(y_test1)
#score = model.evaluate(X_test, y_test,verbose=1)
#report = classification_report( y_test1, y_pred1 )
#print(report)
#cm = confusion_matrix(y_test1, y_pred1)
#print(cm)
#precision_score(y_test, y_pred)
#print(score)

word2vec test

aimport gensim.models.keyedvectors as word2vec
import numpy as np

model=word2vec.KeyedVectors.load_word2vec_format('./model/GoogleNews-vectors-negative300.bin',binary=True)
def get_score(tokens,ground_truth):
    sc2=[]
    for tk in tokens:
        i =0
        sc=[]
        for tk_2 in ground_truth:
            sim =model.similarity(tk,tk_2)
        #print(i, sim)
            sc.append(sim)
            i = i + 1
        sc=np.array(sc)
        sc2.append(np.max(sc,axis=0))
    return sc2
def get_softP(pred,tokens,tokens_scores):
    i=1
    sum1=0
    for (tk,tk_score) in zip(tokens,token_scores):
        #print (tk, tk_score)
         if tk in pred:
            sum1= sum1 + tk_score
            i = i +1
            print (tk, tk_score)
        #print ("Sum is ",sum1)
    #print sum1/i
    return sum1/i

def get_softR(pred,ground_truth,tokens,tokens_scores):
   
    sum1=0
    for (tk,tk_score) in zip(tokens,token_scores):
        #print (tk, tk_score)
        if tk in pred:
             if tk in ground_truth:
                sum1= sum1 + tk_score
                print (tk, tk_score)
        #print ("Sum is ",sum1)
    #print sum1/i
    return sum1/len(ground_truth)

ground_truth=['Finland','University','UEF','Joensuu']
tokens = ['Finland','Departmet','School','Computing','University', 'UEF', 'Science', 'Park', 'Joensuu']
pred=['Finland','School','Park','Joensuu','Computing']

token_scores= get_score(tokens,ground_truth)
get_softP(pred,tokens,token_scores)
get_softR(pred,ground_truth,tokens,token_scores)