如何使用neo4j和transformer构建知识图谱-金马国际

!pip install -u pip setuptools wheel!python -m spacy project clone tutorials/rel_component!pip install -u spacy-nightly --pre!!pip install -u spacy transformersimport spacy#安装完依赖项后重启运行时nlp = spacy.load("[path_to_the_model]/model-best")

import pandas as pddef get_all_documents():df = pd.read_csv("/content/drive/mydrive/job_db1_1_29.csv",sep='"',header=none)documents = []for index,row in df.iterrows():documents.append(str(row[0]))return documentsdocuments = get_all_documents()documents = documents[:]

import hashlibdef extract_ents(documents,nlp):  docs = list()  for doc in nlp.pipe(documents, disable=["tagger", "parser"]):      dictionary=dict.fromkeys(["text", "annotations"])      dictionary["text"]= str(doc)      dictionary['text_sha256'] =  hashlib.sha256(dictionary["text"].encode('utf-8')).hexdigest()      annotations=[]
for e in doc.ents:        ent_id = hashlib.sha256(str(e.text).encode('utf-8')).hexdigest()        ent = {"start":e.start_char,"end":e.end_char, "label":e.label_,"label_upper":e.label_.upper(),"text":e.text,"id":ent_id}        if e.label_ == "experience":          ent["years"] = int(e.text[0])        annotations.append(ent)
dictionary["annotations"] = annotations      docs.append(dictionary)  #print(annotations)  return docsparsed_ents = extract_ents(documents,nlp)

[('stock market analysis', 'skills'),('private investor', 'skills'), ('c  ', 'skills'), ('investment software', 'skills'),('ms windows', 'skills'), ('web development', 'skills'), ('computer science', 'diploma_major'),('ai', 'skills'),('software development', 'skills'),('coding', 'skills'),('c', 'skills'), ('c  ', 'skills'),('visual studio', 'skills'),('2 years', 'experience'), ('c/c   development', 'skills'), ('data compression', 'skills'),('financial markets', 'skills'),('financial calculation', 'skills'),('gui design', 'skills'),('windows development', 'skills'), ('mfc', 'skills'), ('win', 'skills'),('http', 'skills'),('tcp/ip', 'skills'),('sockets', 'skills'), ('network programming', 'skills'), ('system administration', 'skills')]

cd rel_component/

import randomimport typerfrom pathlib import pathimport spacyfrom spacy.tokens import docbin, docfrom spacy.training.example import example#使factory生效from rel_pipe import make_relation_extractor, score_relations#使config生效from rel_model import create_relation_model, create_classification_layer, create_instances, create_tensors#安装完依赖项后重启运行时nlp2 = spacy.load("/content/drive/mydrive/training_rel_roberta/model-best")def extract_relations(documents,nlp,nlp2): predicted_rels = list()for doc in nlp.pipe(documents, disable=["tagger", "parser"]): source_hash = hashlib.sha256(doc.text.encode('utf-8')).hexdigest()for name, proc in nlp2.pipeline: doc = proc(doc)for value, rel_dict in doc._.rel.items():for e in doc.ents:for b in doc.ents:if e.start == value[0] and b.start == value[1]: max_key = max(rel_dict, key=rel_dict. get)#print(max_key) e_id = hashlib.sha256(str(e).encode('utf-8')).hexdigest() b_id = hashlib.sha256(str(b).encode('utf-8')).hexdigest()if rel_dict[max_key] >=0.9 :#print(f" entities: {e.text, b.text} --> predicted relation: {rel_dict}") predicted_rels.append({'head': e_id, 'tail': b_id, 'type':max_key, 'source': source_hash})return predicted_relspredicted_rels = extract_relations(documents,nlp,nlp2)

predicted relations:  entities: ('5  years', 'software engineering') --> predicted relation: {'degree_in': 9.5471655e-08, 'experience_in': 0.9967771} entities: ('5  years', 'technical management') --> predicted relation: {'degree_in': 1.1285037e-07, 'experience_in': 0.9961034}  entities: ('5  years', 'designing') --> predicted relation:{'degree_in': 1.3603304e-08, 'experience_in': 0.9989103}  entities: ('4  years', 'performance management') --> predicted relation: {'degree_in': 6.748373e-08, 'experience_in': 0.92884386}

documents = get_all_documents()documents = documents[:]parsed_ents = extract_ents(documents,nlp)predicted_rels = extract_relations(documents,nlp,nlp2)#neo4j的基础查询功能from neo4j import graphdatabaseimport pandas as pdhost = 'bolt://[your_host_address]'user = 'neo4j'password = '[your_password]'driver = graphdatabase.driver(host,auth=(user, password))def neo4j_query(query, params=none):with driver.session() as session: result = session.run(query, params)return pd.dataframe([r.values() for r in result], columns=result.keys())

#清空当前的neo4j sandbox db (删除所有东西)neo4j_query("""match (n) detach delete n;""")#创建第一个主节点neo4j_query("""merge (l:labormarket {name:"labor market"})return l""")#向kg中添加实体：技能、经验、学历、专业neo4j_query("""match (l:labormarket)unwind $data as rowmerge (o:offer{id:row.text_sha256})set o.text = row.textmerge (l)-[:has_offer]->(o)with o, row.annotations as entitiesunwind entities as entitymerge (e:entity {id:entity.id})on create set e.name = entity.text, e.label = entity.label_uppermerge (o)-[m:mentions]->(e)on create set m.count = 1on match set m.count = m.count   1with e as ecall apoc.create.addlabels( id(e), [ e.label ] )yield noderemove node.labelreturn node""", {'data': parsed_ents})#为实体experience添加属性'name'res = neo4j_query("""match (e:experience)return e.id as id, e.name as name""")#从experience name中提取工作年限，并保存在属性years中import redef get_years(name):return re.findall(r"\d ",name)[0]res["years"] = res.name.map(lambda name: get_years(name))data = res.to_dict('records')#为实体experience添加属性'years'neo4j_query("""unwind $data as rowmatch (e:experience {id:row.id})set e.years = row.yearsreturn e.name as name, e.years as years""",{"data":data})#将关系添加到kgneo4j_query("""unwind $data as rowmatch (source:entity {id: row.head})match (target:entity {id: row.tail})match (offer:offer {id: row.source})merge (source)-[:rel]->(r:relation {type: row.type})-[:rel]->(target)merge (offer)-[:mentions]->(r)""", {'data': predicted_rels})

#在表中显示最佳匹配项other_id = "8de6e42ddfbc2a8bd7008d93516c57e50fa815e64e387eb2fc7a27000ae904b6"query = """match (o1:offer {id:$id})-[m1:mentions]->(s:entity)<- [m2:mentions]-(o2:offer)return distinct o1.id as source,o2.id as proposed_offer, count(*) as freq, collect(s.name) as common_termsorder by freqdesc limit $limit"""res = neo4j_query(query,{"id":other_id,"limit":3})res#在neo4j浏览器中，使用该查询显示最佳匹配项的图"""match (o1:offer {id:"8de6e42ddfbc2a8bd7008d93516c57e50fa815e64e387eb2fc7a27000ae904b6"})-[m1:mentions]->(s:entity)<- [m2:mentions]-(o2:offer)with o1,s,o2, count(*) as freqmatch (o1)--(s)return collect(o2)[0], o1,s, max(freq)"""

query = """match (s:skills)<-[:mentions]-(o:offer)return s.name as skill, count(o) as freqorder by freq desclimit 10"""res = neo4j_query(query)res

query = """match (s:skills)--(r:relation)--(e:experience) where r.type = "experience_in"return s.name as skill,e.years as yearsorder by years desclimit 10"""res = neo4j_query(query)res

neo4j_query("""match (s1:skills)<-[:mentions]-(:offer)-[:mentions]->(s2:skills)where id(s1) < id(s2)return s1.name as skill1, s2.name as skill2, count(*) as cooccurrenceorder by cooccurrencedesc limit 5""")