Mentions légales du service

Skip to content
Snippets Groups Projects
Commit cd1ea771 authored by Hugo Ayats's avatar Hugo Ayats
Browse files

testing & debugging

parent 71e3f3b4
No related branches found
No related tags found
No related merge requests found
<component name="ArtifactManager">
<artifact type="jar" build-on-make="true" name="conceptualKNN:jar">
<output-path>$PROJECT_DIR$/out/artifacts/conceptualKNN_jar</output-path>
<root id="archive" name="Jena_conceptsOfNeighbours.jar">
<element id="module-output" name="Jena_conceptsOfNeighbours" />
<element id="extracted-dir" path="$MAVEN_REPOSITORY$/org/apache/jena/jena-core/3.16.0/jena-core-3.16.0.jar" path-in-jar="/" />
<element id="extracted-dir" path="$MAVEN_REPOSITORY$/org/apache/jena/jena-base/3.16.0/jena-base-3.16.0.jar" path-in-jar="/" />
<element id="extracted-dir" path="$MAVEN_REPOSITORY$/org/slf4j/slf4j-api/1.7.30/slf4j-api-1.7.30.jar" path-in-jar="/" />
<element id="extracted-dir" path="$MAVEN_REPOSITORY$/org/apache/jena/jena-shaded-guava/3.16.0/jena-shaded-guava-3.16.0.jar" path-in-jar="/" />
<element id="extracted-dir" path="$MAVEN_REPOSITORY$/org/apache/commons/commons-csv/1.8/commons-csv-1.8.jar" path-in-jar="/" />
<element id="extracted-dir" path="$MAVEN_REPOSITORY$/commons-io/commons-io/2.6/commons-io-2.6.jar" path-in-jar="/" />
<element id="extracted-dir" path="$MAVEN_REPOSITORY$/org/apache/commons/commons-lang3/3.10/commons-lang3-3.10.jar" path-in-jar="/" />
<element id="extracted-dir" path="$MAVEN_REPOSITORY$/commons-codec/commons-codec/1.14/commons-codec-1.14.jar" path-in-jar="/" />
<element id="extracted-dir" path="$MAVEN_REPOSITORY$/org/apache/commons/commons-compress/1.20/commons-compress-1.20.jar" path-in-jar="/" />
<element id="extracted-dir" path="$MAVEN_REPOSITORY$/com/github/andrewoma/dexx/collection/0.7/collection-0.7.jar" path-in-jar="/" />
<element id="extracted-dir" path="$MAVEN_REPOSITORY$/org/apache/jena/jena-iri/3.16.0/jena-iri-3.16.0.jar" path-in-jar="/" />
<element id="extracted-dir" path="$MAVEN_REPOSITORY$/commons-cli/commons-cli/1.4/commons-cli-1.4.jar" path-in-jar="/" />
<element id="extracted-dir" path="$MAVEN_REPOSITORY$/org/apache/jena/jena-arq/3.16.0/jena-arq-3.16.0.jar" path-in-jar="/" />
<element id="extracted-dir" path="$MAVEN_REPOSITORY$/org/apache/httpcomponents/httpclient/4.5.10/httpclient-4.5.10.jar" path-in-jar="/" />
<element id="extracted-dir" path="$MAVEN_REPOSITORY$/org/apache/httpcomponents/httpcore/4.4.12/httpcore-4.4.12.jar" path-in-jar="/" />
<element id="extracted-dir" path="$MAVEN_REPOSITORY$/com/github/jsonld-java/jsonld-java/0.12.5/jsonld-java-0.12.5.jar" path-in-jar="/" />
<element id="extracted-dir" path="$MAVEN_REPOSITORY$/com/fasterxml/jackson/core/jackson-core/2.10.1/jackson-core-2.10.1.jar" path-in-jar="/" />
<element id="extracted-dir" path="$MAVEN_REPOSITORY$/com/fasterxml/jackson/core/jackson-databind/2.10.1/jackson-databind-2.10.1.jar" path-in-jar="/" />
<element id="extracted-dir" path="$MAVEN_REPOSITORY$/com/fasterxml/jackson/core/jackson-annotations/2.10.1/jackson-annotations-2.10.1.jar" path-in-jar="/" />
<element id="extracted-dir" path="$MAVEN_REPOSITORY$/org/apache/httpcomponents/httpclient-cache/4.5.10/httpclient-cache-4.5.10.jar" path-in-jar="/" />
<element id="extracted-dir" path="$MAVEN_REPOSITORY$/org/apache/thrift/libthrift/0.13.0/libthrift-0.13.0.jar" path-in-jar="/" />
<element id="extracted-dir" path="$MAVEN_REPOSITORY$/org/slf4j/slf4j-api/1.7.25/slf4j-api-1.7.25.jar" path-in-jar="/" />
<element id="extracted-dir" path="$MAVEN_REPOSITORY$/javax/annotation/javax.annotation-api/1.3.2/javax.annotation-api-1.3.2.jar" path-in-jar="/" />
<element id="extracted-dir" path="$MAVEN_REPOSITORY$/org/slf4j/jcl-over-slf4j/1.7.30/jcl-over-slf4j-1.7.30.jar" path-in-jar="/" />
</root>
</artifact>
</component>
\ No newline at end of file
......@@ -149,4 +149,8 @@ public class ConceptOfNeighbours implements Jsonable {
res += "\n}";
return res;
}
public int getExtensionalDistance() {
return extensionalDistance;
}
}
package conceptsOfNeighbours;
import conceptsOfNeighbours.matchTree.LazyJoinResults;
import conceptsOfNeighbours.matchTree.MatchTree;
import conceptsOfNeighbours.utils.*;
import org.apache.jena.graph.Node;
......@@ -32,6 +33,7 @@ public class Partition implements Jsonable {
private final Map<Node, Var> varsByUri = new HashMap<>(); // map uris and literals to vars
private final Map<Node, Integer> uriDepth = new HashMap<>();
private int nextKey = 0; // for var numbering
private int partitioningSteps = 0;
/**
* Standard constructor
......@@ -70,11 +72,10 @@ public class Partition implements Jsonable {
/**
* Used to set the depth of a node, i.e the distance from the target node to this node. Shouldn't be used by the user.
*/
public int setDepth(Node node, int depth) {
public void setDepth(Node node, int depth) {
if(!this.uriDepth.containsKey(node) || this.uriDepth.get(node) > depth) {
this.uriDepth.put(node, depth);
}
return getDepth(node);
}
/**
......@@ -88,15 +89,6 @@ public class Partition implements Jsonable {
return this.model;
}
/**
* Give the arity of the concepts of neighbours
*
* @return the arity as an integer
*/
private int getArity() {
return this.target.size();
}
private int getNextClusterId() {
this.nextClusterId++;
return nextClusterId;
......@@ -140,7 +132,10 @@ public class Partition implements Jsonable {
* @return The table representing the answers
*/
public Table getAnswer(Element elt) {
return this.answers.getOrDefault(elt, null);
if(this.answers.containsKey(elt)) {
return this.answers.get(elt);
}
return null;
}
/**
......@@ -155,6 +150,8 @@ public class Partition implements Jsonable {
throw new PartitionException("Tried to resume finished partition");
}
this.partitioningSteps++;
// Selection of the pre-concept to refine
ConceptOfNeighbours preConcept = this.preConcepts.remove(0); // The concept to refine
......@@ -170,7 +167,11 @@ public class Partition implements Jsonable {
do {
elt = iter.next();
mentionedVars = ElementUtils.mentioned(elt);
} while(!preConcept.connected(mentionedVars));
} while(iter.hasNext() && !preConcept.connected(mentionedVars));
if(!preConcept.connected(mentionedVars)) {
this.concepts.add(preConcept);
return this.preConcepts.isEmpty();
}
// Intensions of the new (pre-)concepts
Set<Element> intensionWithoutElt = preConcept.getIntension();
......@@ -181,7 +182,9 @@ public class Partition implements Jsonable {
MatchTree matchTreeWithoutElt = preConcept.getMatchTree();
MatchTree matchTreeWithElt = new MatchTree(matchTreeWithoutElt);
try {
matchTreeWithElt = matchTreeWithElt.lazyJoin(new MatchTree(elt, this, preConcept.getConnectedVars())).getMatchTree();
MatchTree newMatchTree = new MatchTree(elt, this, preConcept.getConnectedVars());
LazyJoinResults result = matchTreeWithElt.lazyJoin(newMatchTree);
matchTreeWithElt = result.getMatchTree();
} catch(OutOfMemoryError exception) {
this.preConcepts.add(preConcept);
throw exception;
......@@ -209,7 +212,7 @@ public class Partition implements Jsonable {
Set<Var> connectedVarsWithElt = new HashSet<>(preConcept.getConnectedVars());
connectedVarsWithElt.addAll(ElementUtils.mentioned(elt));
int extDistWithElt = matchTreeWithElt.getMatchSet().size();
int extDistWithElt = TableUtils.projection(matchTreeWithElt.getMatchSet(), new HashSet<>(preConcept.getProjectionVars())).size();
ConceptOfNeighbours preConceptWithElt = new ConceptOfNeighbours(
this.getNextClusterId(),
......@@ -292,7 +295,7 @@ public class Partition implements Jsonable {
public String toJson() {
StringBuilder res = new StringBuilder();
StringBuilder uris = new StringBuilder();
for(String uri: this.target) {
for(String uri : this.target) {
if(uris.toString().equals("")) {
uris.append("[");
} else {
......@@ -302,18 +305,33 @@ public class Partition implements Jsonable {
}
uris.append("]");
res.append("{\n\"target\":").append(uris.toString()).append(",\n");
res.append("\"clusters\":[\n\t");
PriorityQueue<ConceptOfNeighbours> queue = new PriorityQueue<>(this.preConcepts);
while(!queue.isEmpty()) {
res.append("\t").append(queue.poll().toJson().replaceAll("\n", "\n\t").replaceAll("\"\",", ""));
res.append(",\n");
}
queue = new PriorityQueue<>(this.concepts);
while(!queue.isEmpty()) {
res.append("\t").append(queue.poll().toJson().replaceAll("\n", "\n\t").replaceAll("\"\",", ""));
if(!queue.isEmpty()) res.append(",\n");
}
res.append("\"conceptsOfNeighbours\":[\n\t");
List<ConceptOfNeighbours> concepts = new ArrayList<>();
concepts.addAll(this.preConcepts);
concepts.addAll(this.concepts);
concepts.sort(Comparator.comparingInt(ConceptOfNeighbours::getExtensionalDistance));
StringBuilder conceptsString = new StringBuilder();
concepts.forEach(c -> {
if(!conceptsString.toString().equals("")) {
conceptsString.append(",");
}
conceptsString.append("\t").append(c.toJson().replaceAll("\n", "\n\t").replaceAll("\"\",", ""));
});
res.append(conceptsString);
res.append("]\n}");
return res.toString();
}
public int getNbConcepts() {
return this.concepts.size();
}
public int getNbPreConcepts() {
return this.preConcepts.size();
}
public int getPartitioningSteps() {
return partitioningSteps;
}
}
......@@ -77,7 +77,11 @@ public class MatchTree {
*/
public MatchTree(MatchTree matchTree) {
this.element = matchTree.element;
this.eltVars = ElementUtils.mentioned(this.element);
if(this.element != null) {
this.eltVars = ElementUtils.mentioned(this.element);
} else {
this.eltVars = new HashSet<>();
}
this.newVars = matchTree.newVars;
this.matchTable = matchTree.matchTable;
......@@ -120,6 +124,7 @@ public class MatchTree {
copy = new MatchTree(this);
copy.children.remove(child);
copy.children.add(childResult.getMatchTree());
child = childResult.getMatchTree();
copy.matchTable = TableUtils.join(this.matchTable, TableUtils.projection(child.matchTable, child.propagationVars));
}
......@@ -127,7 +132,7 @@ public class MatchTree {
Set<Var> variablesDefinedInNode = new HashSet<>(this.newVars);
variablesDefinedInNode.retainAll(newMatchTree.propagationVars);
if(variablesDefinedInNode.isEmpty()) {
if(!variablesDefinedInNode.isEmpty()) {
isModified = true;
if(copy == null) {
copy = new MatchTree(this);
......@@ -148,8 +153,13 @@ public class MatchTree {
propagationVarsPlus.removeAll(propagationVarsMinus);
propagationVarsMinus.removeAll(propagationVarsPlus);
this.propagationVars.addAll(propagationVarsMinus);
this.propagationVars.addAll(propagationVarsPlus);
if(isModified) {
copy.propagationVars.addAll(propagationVarsMinus);
copy.propagationVars.addAll(propagationVarsPlus);
} else {
this.propagationVars.addAll(propagationVarsMinus);
this.propagationVars.addAll(propagationVarsPlus);
}
return new LazyJoinResults(isModified, propagationVarsMinus, propagationVarsPlus, copy);
}
......
......@@ -15,7 +15,6 @@ import org.apache.jena.sparql.expr.*;
import org.apache.jena.sparql.expr.nodevalue.NodeValueNode;
import org.apache.jena.sparql.syntax.Element;
import org.apache.jena.sparql.syntax.ElementFilter;
import org.apache.jena.sparql.syntax.ElementPathBlock;
import org.apache.jena.sparql.syntax.ElementTriplesBlock;
import org.apache.jena.vocabulary.RDF;
import org.apache.jena.vocabulary.RDFS;
......@@ -74,24 +73,24 @@ public class ElementUtils {
res.addBinding(BindingFactory.binding(var, node));
}
} else if(element instanceof ElementPathBlock) {// If the element is a triple
Var subject = (Var) (((ElementPathBlock) element).getPattern().get(0).getSubject()); // necessarily a var, as constants are managed through filters
Property property = ResourceFactory.createProperty(((ElementPathBlock) element).getPattern().get(0).getPredicate().getURI());
Node object = ((ElementPathBlock) element).getPattern().get(0).getObject();
} else if(element instanceof ElementTriplesBlock) {// If the element is a triple
Var subject = (Var) (((ElementTriplesBlock) element).getPattern().get(0).getSubject()); // necessarily a var, as constants are managed through filters
Property property = ResourceFactory.createProperty(((ElementTriplesBlock) element).getPattern().get(0).getPredicate().getURI());
Node object = ((ElementTriplesBlock) element).getPattern().get(0).getObject();
if(object.isVariable()){ // most of the time
if(object.isVariable()) { // most of the time
Var objVar = (Var) object;
model.getModel().listSubjectsWithProperty(property).forEachRemaining(subj ->
model.getModel().listObjectsOfProperty(subj, property).forEachRemaining(obj ->
res.addBinding(BindingFactory.binding(BindingFactory.binding(subject, subj.asNode()), objVar, obj.asNode()))
)
model.getModel().listObjectsOfProperty(subj, property).forEachRemaining(obj ->
res.addBinding(BindingFactory.binding(BindingFactory.binding(subject, subj.asNode()), objVar, obj.asNode()))
)
);
} else { // only happens for types and ontology triples
RDFNode rdfObject = ResourceFactory.createResource(object.getURI());
model.getModel().listSubjectsWithProperty(property, rdfObject).forEachRemaining(subj ->
res.addBinding(BindingFactory.binding(subject, subject.asNode()))
res.addBinding(BindingFactory.binding(subject, subj.asNode()))
);
}
}
......@@ -129,7 +128,7 @@ public class ElementUtils {
model.getTriples().get(node).forEach((property, objects) -> { // describe the node by the triples having the node as subject
if(property.equals(RDF.type)) { // use the types as labels
objects.forEach(rdfNode -> {
ElementPathBlock tripleElement = new ElementPathBlock();
ElementTriplesBlock tripleElement = new ElementTriplesBlock();
tripleElement.addTriple(Triple.create(partition.getOrCreateVar(node), property.asNode(), rdfNode));
description.add(tripleElement);
});
......@@ -137,7 +136,7 @@ public class ElementUtils {
objects.forEach(object -> {
partition.setDepth(object, nodeDepth + 1);
Var var = partition.getOrCreateVar(object);
ElementPathBlock triple = new ElementPathBlock();
ElementTriplesBlock triple = new ElementTriplesBlock();
triple.addTriple(Triple.create(partition.getOrCreateVar(node), property.asNode(), var));
ElementFilter filter = new ElementFilter((new E_Equals(new ExprVar(var), new NodeValueNode(object))));
description.add(triple);
......@@ -151,8 +150,9 @@ public class ElementUtils {
model.getReversedTriples().get(node).forEach((property, subjects) -> { // describe the node by the triples having the node as object
if(!property.equals(RDF.type)) { // avoid to describe the classes by their members
subjects.forEach(subject -> {
partition.setDepth(subject, nodeDepth + 1);
Var var = partition.getOrCreateVar(subject);
ElementPathBlock triple = new ElementPathBlock();
ElementTriplesBlock triple = new ElementTriplesBlock();
triple.addTriple(Triple.create(var, property.asNode(), partition.getOrCreateVar(node)));
ElementFilter filter = new ElementFilter(new E_Equals(new ExprVar(var), new NodeValueNode(subject)));
description.add(triple);
......@@ -239,7 +239,7 @@ public class ElementUtils {
if(!res.toString().equals("[")) {
res.append(",");
}
res.append(e.toString().replaceAll("[^\\\\]\"", "\\\\\""));
res.append("\"").append(e.toString().replaceAll("[^\\\\]\"", "\\\\\"")).append("\"");
}
res.append("]");
return res.toString();
......
package conceptsOfNeighbours.utils;
import conceptsOfNeighbours.Partition;
import java.lang.management.ManagementFactory;
import java.util.concurrent.atomic.AtomicBoolean;
public class PartitioningThread extends Thread {
private final Partition partition;
private final AtomicBoolean cut;
private long execTime;
public PartitioningThread(Partition partition, AtomicBoolean cut) {
super();
this.partition = partition;
this.cut = cut;
}
@Override
public void run() {
try {
this.partition.fullPartitioning(cut);
} catch(PartitionException | TableException e) {
e.printStackTrace();
}
this.execTime = ManagementFactory.getThreadMXBean().getThreadCpuTime(this.getId());
}
public long getExecTime() {
return execTime;
}
}
......@@ -5,6 +5,7 @@ import org.apache.jena.sparql.algebra.TableFactory;
import org.apache.jena.sparql.core.Var;
import org.apache.jena.sparql.engine.binding.Binding;
import org.apache.jena.sparql.engine.binding.BindingFactory;
import org.apache.jena.sparql.engine.binding.BindingMap;
import org.apache.jena.sparql.engine.join.QueryIterHashJoin;
import java.util.Iterator;
......@@ -28,21 +29,23 @@ public class TableUtils {
throw new TableException("Tried to project a table on a non-existing variable");
}
Table projection;
if(vars.containsAll(table.getVars())) {
return table;
}
Table proj = TableFactory.create();
table.rows().forEachRemaining(tuple -> {
Binding binding = BindingFactory.binding();
projection = TableFactory.create();
Iterator<Binding> iter = table.rows();
Binding b;
while(iter.hasNext()) {
b = iter.next();
BindingMap bind = BindingFactory.create();
for(Var var : vars) {
binding = BindingFactory.binding(binding, var, tuple.get(var));
bind.add(var, b.get(var));
}
if(!proj.contains(binding)) {
proj.addBinding(binding);
}
});
return proj;
projection.addBinding(bind);
}
return projection;
}
/**
......@@ -54,11 +57,11 @@ public class TableUtils {
*/
public static Table intersect(Table table1, Table table2) throws TableException {
// Check table compatibility
if(table1.getVars().equals(table2.getVars())) {
if(!table1.getVars().equals(table2.getVars())) {
throw new TableException("Tried to intersect two tables of different schemas");
}
Table inter = TableFactory.create();
Table inter = TableFactory.create(table1.getVars());
table1.rows().forEachRemaining(row ->
{
if(table2.contains(row)) {
......@@ -77,11 +80,11 @@ public class TableUtils {
*/
public static Table difference(Table table1, Table table2) throws TableException {
// Check table compatibility
if(table1.getVars().equals(table2.getVars())) {
throw new TableException("Tried to intersect two tables of different schemas");
if(!table1.getVars().equals(table2.getVars())) {
throw new TableException("Tried to compute the difference two tables of different schemas");
}
Table diff = TableFactory.create();
Table diff = TableFactory.create(table1.getVars());
table1.rows().forEachRemaining(row -> {
if(!table2.contains(row)) {
diff.addBinding(row);
......@@ -144,7 +147,7 @@ public class TableUtils {
for(Iterator<Var> it2 = b.vars(); it2.hasNext(); ) {
Var v = it2.next();
line.append(b.get(v).toString());
line.append("\"").append(b.get(v).toString()).append("\"");
if(it2.hasNext()) {
line.append(",");
}
......
package test;
import conceptsOfNeighbours.CnnModel;
import conceptsOfNeighbours.Partition;
import conceptsOfNeighbours.utils.PartitioningThread;
import org.apache.jena.rdf.model.InfModel;
import org.apache.jena.rdf.model.ModelFactory;
import org.apache.jena.reasoner.ReasonerRegistry;
import org.apache.jena.sparql.algebra.Table;
import org.apache.jena.sparql.algebra.TableFactory;
import org.apache.jena.sparql.core.Var;
import org.apache.jena.sparql.engine.binding.Binding;
import org.apache.jena.sparql.engine.binding.BindingFactory;
import java.io.FileWriter;
import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
import java.util.concurrent.atomic.AtomicBoolean;
public class Test {
public static void main(String[] args) throws IOException, InterruptedException {
String input = args[0];
String output = args[1];
int exec_time = 5;
String filter = "mondial/10/river";
boolean filterTable = true;
InfModel rdfModel = ModelFactory.createInfModel(ReasonerRegistry.getRDFSReasoner(), ModelFactory.createDefaultModel());
rdfModel.read(input, "TTL");
CnnModel model = new CnnModel(rdfModel);
List<String> target = new ArrayList<>();
target.add("http://www.semwebtech.org/mondial/10/river/Loire/");
Table table = TableFactory.create();
Var var = Var.alloc("Neighbor_0");
rdfModel.listSubjects().forEachRemaining(r -> {
if(!filterTable || r.getURI().contains(filter)) {
Binding b = BindingFactory.binding(var, r.asNode());
table.addBinding(b);
}
});
rdfModel.listObjects().forEachRemaining(r -> {
if(!filterTable || r.isURIResource() && r.asResource().getURI().contains(filter)) {
Binding b = BindingFactory.binding(var, r.asNode());
if(!table.contains(b)) {
table.addBinding(b);
}
}
});
System.out.printf("Initialization table size: %d\n", table.size());
Partition partition = new Partition(model, target, table, 2);
AtomicBoolean cut = new AtomicBoolean(false);
PartitioningThread thread = new PartitioningThread(partition, cut);
thread.start();
thread.join(exec_time * 1000);
cut.set(true);
System.out.println("Waiting for the thread to finish");
if(thread.isAlive()) {
thread.join();
}
System.out.printf("Theoretical max exec time: %d seconds\n", exec_time);
System.out.printf("Real exec time: %f seconds\n", (float) thread.getExecTime() / 1000000000);
int nbConcepts = partition.getNbConcepts();
int nbPreConcepts = partition.getNbPreConcepts();
System.out.printf("Nb of concepts: %d + %d = %d\n", nbConcepts, nbPreConcepts, nbConcepts + nbPreConcepts);
System.out.printf("Nb of partitioning steps: %d\n", partition.getPartitioningSteps());
FileWriter writer = new FileWriter(output);
writer.write(partition.toJson());
writer.close();
}
}
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Please register or to comment