Skip to content

Commit

Permalink
refactor and new "raw" dsl to include open constraints
Browse files Browse the repository at this point in the history 
Signed-off-by: Jorge Aguilera <[email protected]>
  • Loading branch information
@jagedn
jagedn committed Jul 15, 2024
1 parent c110afc commit 6e7cebf
Show file tree
Hide file tree
Showing 5 changed files with 162 additions and 140 deletions.
74 changes: 12 additions & 62 deletions plugins/nf-nomad/src/main/nextflow/nomad/models/ConstraintsBuilder.groovy
View file
Original file line number Diff line number Diff line change
Expand Up @@ -25,72 +25,22 @@ class ConstraintsBuilder {
}

protected static List<Constraint> nodeConstraints(JobConstraintsNode nodeSpec){
def ret = [] as List<Constraint>
if( nodeSpec.id ){
ret.add new Constraint()
.ltarget('${node.unique.id}')
.operand("=")
.rtarget(nodeSpec.id)
}
if( nodeSpec.name ){
ret.add new Constraint()
.ltarget('${node.unique.name}')
.operand("=")
.rtarget(nodeSpec.name)
}
if( nodeSpec.clientClass ){
ret.add new Constraint()
.ltarget('${node.class}')
.operand("=")
.rtarget(nodeSpec.clientClass)
}
if( nodeSpec.dataCenter ){
ret.add new Constraint()
.ltarget('${node.datacenter}')
.operand("=")
.rtarget(nodeSpec.dataCenter)
}
if( nodeSpec.region ){
ret.add new Constraint()
.ltarget('${node.region}')
.operand("=")
.rtarget(nodeSpec.region)
}
if( nodeSpec.pool ){
ret.add new Constraint()
.ltarget('${node.pool}')
.operand("=")
.rtarget(nodeSpec.pool)
}
def ret = nodeSpec.raws?.collect{ triple->
return new Constraint()
.ltarget('${'+triple.left+'}')
.operand(triple.middle)
.rtarget(triple.right)
} as List<Constraint>
ret
}

protected static List<Constraint> attrConstraints(JobConstraintsAttr nodeSpec) {
def ret = [] as List<Constraint>
if (nodeSpec.arch) {
ret.add new Constraint()
.ltarget('${attr.cpu.arch}')
.operand("=")
.rtarget(nodeSpec.arch)
}
if (nodeSpec.numcores) {
ret.add new Constraint()
.ltarget('${attr.cpu.numcores}')
.operand("=")
.rtarget("$nodeSpec.numcores")
}
if (nodeSpec.reservablecores) {
ret.add new Constraint()
.ltarget('${attr.cpu.reservablecores}')
.operand("=")
.rtarget("$nodeSpec.reservablecores")
}
if (nodeSpec.totalcompute) {
ret.add new Constraint()
.ltarget('${attr.cpu.totalcompute}')
.operand("=")
.rtarget("$nodeSpec.totalcompute")
}
def ret = nodeSpec.raws?.collect{ triple->
return new Constraint()
.ltarget('${'+triple.left+'}')
.operand(triple.middle)
.rtarget(triple.right)
} as List<Constraint>
ret
}

Expand Down
59 changes: 40 additions & 19 deletions plugins/nf-nomad/src/main/nextflow/nomad/models/JobConstraintsAttr.groovy
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,8 @@

package nextflow.nomad.models

import org.apache.commons.lang3.tuple.Triple

/**
* Nomad Job Constraint Spec
*
Expand All @@ -26,37 +28,56 @@ package nextflow.nomad.models

class JobConstraintsAttr {

private String arch = null
private Integer numcores= null
private Integer reservablecores= null
private Double totalcompute= null
private List<Triple<String, String, String>> raws= []

String getArch() {
return arch
List<Triple<String, String, String>> getRaws() {
return raws
}

Integer getNumcores() {
return numcores
JobConstraintsAttr setCpu(Map map){
cpu(map)
}

Integer getReservablecores() {
return reservablecores
JobConstraintsAttr cpu(Map map){
if( map.containsKey('arch'))
raw("cpu.arch","=", map['arch'].toString())
if( map.containsKey('numcores'))
raw("cpu.numcores",">=", map['numcores'].toString())
if( map.containsKey('reservablecores'))
raw("cpu.reservablecores",">=", map['reservablecores'].toString())
if( map.containsKey('totalcompute'))
raw("cpu.totalcompute","=", map['totalcompute'].toString())
this
}

Double getTotalcompute() {
return totalcompute
JobConstraintsAttr setUnique(Map map){
unique(map)
}

JobConstraintsAttr setCpu(Map map){
cpu(map)
JobConstraintsAttr unique(Map map){
if( map.containsKey('hostname'))
raw("unique.hostname","=", map['hostname'].toString())
if( map.containsKey('ip-address'))
raw("unique.network.ip-address","=", map['ip-address'].toString())
this
}

JobConstraintsAttr cpu(Map map){
this.arch = map.containsKey("arch") ? map["arch"].toString() : null
this.numcores = map.containsKey("numcores") ? map["numcores"] as int : null
this.reservablecores = map.containsKey("reservablecores") ? map["reservablecores"] as int : null
this.totalcompute = map.containsKey("totalcompute") ? map["totalcompute"] as double : null
JobConstraintsAttr setKernel(Map map){
kernel(map)
}

JobConstraintsAttr kernel(Map map){
if( map.containsKey('arch'))
raw("kernel.arch","=", map['arch'].toString())
if( map.containsKey('name'))
raw("kernel.name","=", map['name'].toString())
if( map.containsKey('version'))
raw("kernel.version","=", map['version'].toString())
this
}

JobConstraintsAttr raw(String attr, String operator, String value){
raws.add Triple.of("attr."+attr, operator, value)
this
}
}
58 changes: 21 additions & 37 deletions plugins/nf-nomad/src/main/nextflow/nomad/models/JobConstraintsNode.groovy
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,8 @@

package nextflow.nomad.models

import org.apache.commons.lang3.tuple.Triple

/**
* Nomad Job Constraint Spec
*
Expand All @@ -26,53 +28,30 @@ package nextflow.nomad.models

class JobConstraintsNode {

private String id = null
private String name = null
private String clientClass = null
private String pool = null
private String dataCenter = null
private String region = null

String getId() {
return id
}

String getName() {
return name
}

String getClientClass() {
return clientClass
}

String getPool() {
return pool
}

String getDataCenter() {
return dataCenter
}
private List<Triple<String, String, String>> raws= []

String getRegion() {
return region
List<Triple<String, String, String>> getRaws() {
return raws
}

JobConstraintsNode setUnique(Map map){
unique(map)
}

JobConstraintsNode unique(Map map){
this.id = map.containsKey("id") ? map["id"].toString() : null
this.name = map.containsKey("name") ? map["name"].toString() : null
['id', 'name'].each { key->
if( map.containsKey(key))
raw("unique.${key}","=", map[key].toString())
}
this
}

JobConstraintsNode setClientClass(Object map){
clientClass(map)
JobConstraintsNode setClazz(Object map){ // class is a reserved word, in java we used clazz
clazz(map)
}

JobConstraintsNode clientClass(Object clientClass){
this.clientClass = clientClass.toString()
JobConstraintsNode clazz(Object cls){
raw("class","=", cls.toString())
this
}

Expand All @@ -81,7 +60,7 @@ class JobConstraintsNode {
}

JobConstraintsNode pool(Object pool){
this.pool = pool.toString()
raw("pool","=", pool.toString())
this
}

Expand All @@ -90,7 +69,7 @@ class JobConstraintsNode {
}

JobConstraintsNode dataCenter(Object dataCenter){
this.dataCenter = dataCenter.toString()
raw("datacenter","=", dataCenter.toString())
this
}

Expand All @@ -99,7 +78,12 @@ class JobConstraintsNode {
}

JobConstraintsNode region(Object region){
this.region = region.toString()
raw("region","=", region.toString())
this
}

JobConstraintsNode raw(String attr, String operator, String value){
raws.add Triple.of("node."+attr, operator, value)
this
}
}
44 changes: 22 additions & 22 deletions plugins/nf-nomad/src/test/nextflow/nomad/config/NomadJobConstraints.groovy
View file
Original file line number Diff line number Diff line change
Expand Up @@ -35,7 +35,7 @@ class NomadJobConstraints extends Specification {
constraints: {
node {
unique = [id :"node-id", name: "node-name"]
clientClass = "linux-64bit"
clazz = "linux-64bit"
pool = "custom-pool"
dataCenter = 'dc1'
region = 'us'
Expand All @@ -50,15 +50,15 @@ class NomadJobConstraints extends Specification {
then:
config.jobOpts.constraintsSpec
config.jobOpts.constraintsSpec.nodeSpecs.size() == 1
config.jobOpts.constraintsSpec.nodeSpecs[0].id == "node-id"
config.jobOpts.constraintsSpec.nodeSpecs[0].name == "node-name"
config.jobOpts.constraintsSpec.nodeSpecs[0].clientClass == "linux-64bit"
config.jobOpts.constraintsSpec.nodeSpecs[0].pool == "custom-pool"
config.jobOpts.constraintsSpec.nodeSpecs[0].dataCenter == "dc1"
config.jobOpts.constraintsSpec.nodeSpecs[0].region == "us"
config.jobOpts.constraintsSpec.nodeSpecs[0].raws.find{it.left.endsWith("id")}.right == "node-id"
config.jobOpts.constraintsSpec.nodeSpecs[0].raws.find{it.left.endsWith("name")}.right == "node-name"
config.jobOpts.constraintsSpec.nodeSpecs[0].raws.find{it.left.endsWith("class")}.right == "linux-64bit"
config.jobOpts.constraintsSpec.nodeSpecs[0].raws.find{it.left.endsWith("pool")}.right == "custom-pool"
config.jobOpts.constraintsSpec.nodeSpecs[0].raws.find{it.left.endsWith("datacenter")}.right == "dc1"
config.jobOpts.constraintsSpec.nodeSpecs[0].raws.find{it.left.endsWith("region")}.right == "us"

config.jobOpts.constraintsSpec.attrSpecs.size() == 1
config.jobOpts.constraintsSpec.attrSpecs[0].arch == '286'
config.jobOpts.constraintsSpec.attrSpecs[0].raws[0].right == '286'
}

void "should instantiate a no completed constraints spec"() {
Expand All @@ -68,7 +68,7 @@ class NomadJobConstraints extends Specification {
constraints: {
node {
unique = [id :"node-id", name: "node-name"]
clientClass = "linux-64bit"
clazz = "linux-64bit"
}
}
]
Expand All @@ -77,11 +77,11 @@ class NomadJobConstraints extends Specification {
then:
config.jobOpts.constraintsSpec
config.jobOpts.constraintsSpec.nodeSpecs.size()
config.jobOpts.constraintsSpec.nodeSpecs[0].id == "node-id"
config.jobOpts.constraintsSpec.nodeSpecs[0].name == "node-name"
config.jobOpts.constraintsSpec.nodeSpecs[0].clientClass == "linux-64bit"
!config.jobOpts.constraintsSpec.nodeSpecs[0].pool
!config.jobOpts.constraintsSpec.nodeSpecs[0].dataCenter
config.jobOpts.constraintsSpec.nodeSpecs[0].raws.find{it.left.endsWith("id")}.right == "node-id"
config.jobOpts.constraintsSpec.nodeSpecs[0].raws.find{it.left.endsWith("name")}.right == "node-name"
config.jobOpts.constraintsSpec.nodeSpecs[0].raws.find{it.left.endsWith("class")}.right == "linux-64bit"
!config.jobOpts.constraintsSpec.nodeSpecs[0].raws.find{it.left.endsWith("pool")}
!config.jobOpts.constraintsSpec.nodeSpecs[0].raws.find{it.left.endsWith("datacenter")}
}

void "should instantiate a list of constraints spec if specified"() {
Expand All @@ -91,14 +91,14 @@ class NomadJobConstraints extends Specification {
constraints: {
node {
unique = [id :"node-id", name: "node-name"]
clientClass = "linux-64bit"
clazz = "linux-64bit"
pool = "custom-pool"
dataCenter = 'dc1'
region = 'us'
}
node {
unique = [id :"node-id", name: "node-name"]
clientClass = "linux-64bit"
clazz = "linux-64bit"
pool = "custom-pool"
dataCenter = 'dc1'
region = 'us'
Expand All @@ -110,11 +110,11 @@ class NomadJobConstraints extends Specification {
then:
config.jobOpts.constraintsSpec
config.jobOpts.constraintsSpec.nodeSpecs.size()
config.jobOpts.constraintsSpec.nodeSpecs[0].id == "node-id"
config.jobOpts.constraintsSpec.nodeSpecs[0].name == "node-name"
config.jobOpts.constraintsSpec.nodeSpecs[0].clientClass == "linux-64bit"
config.jobOpts.constraintsSpec.nodeSpecs[0].pool == "custom-pool"
config.jobOpts.constraintsSpec.nodeSpecs[0].dataCenter == "dc1"
config.jobOpts.constraintsSpec.nodeSpecs[0].region == "us"
config.jobOpts.constraintsSpec.nodeSpecs[0].raws.find{it.left.endsWith("id")}.right == "node-id"
config.jobOpts.constraintsSpec.nodeSpecs[0].raws.find{it.left.endsWith("name")}.right == "node-name"
config.jobOpts.constraintsSpec.nodeSpecs[0].raws.find{it.left.endsWith("class")}.right == "linux-64bit"
config.jobOpts.constraintsSpec.nodeSpecs[0].raws.find{it.left.endsWith("pool")}.right == "custom-pool"
config.jobOpts.constraintsSpec.nodeSpecs[0].raws.find{it.left.endsWith("datacenter")}.right == "dc1"
config.jobOpts.constraintsSpec.nodeSpecs[0].raws.find{it.left.endsWith("region")}.right == "us"
}
}
Loading

0 comments on commit 6e7cebf

Please sign in to comment.