Merge branch 'dev' into 'master'

Fix opening /dev/<files> to see if it is a directory (dangerous and it can take too long)

See merge request arm-research/smarter/smarter-device-manager!17

CONTRIBUTING.md

@@ -0,0 +1,16 @@
+# Contributing to SMARTER
+
+SMARTER is an open-source, software platform for the Edge. Contributions are an important part of the platform, and our goal is to make it as simple as possible to become a contributor.
+
+# Project license, and developer certificate of origin
+
+The SMARTER codebase is licensed under the Apache 2.0 open source license, individual components may have their own licenses. 
+Please see the LICENSE file in the SMARTER documentation repository for the full text of this license, or the LICENSE file
+in respective repositories.  New contributions are expected to be lisensed under the Apache 2.0 license or something compatible
+with that license.
+
+Please note that we expect contributors to the project to self-certify that they are authorized to contribute code 
+using the Linux Foundation's Developer Certificate of Origin. See http://developercertificate.org for more details.
+
+Contributors sign-off that they adhere to these requirements by adding a Signed-off-by line to commit messages.
+Pull requests for contributions without a signed-off-by line will not be accepted.

CONTRIBUTIONS

@@ -1,4 +0,0 @@
-We sincerely appreciate your interest of contributing to this project but for now
-we are not able to accept contributions. We expect to change this policy soon and
-provide a contribution procedure. We suggest that the project be maintained on a
-different branch until the contribution policy is changed.

Dockerfile

@@ -5,11 +5,12 @@ RUN apk update && apk upgrade && apk add tar ca-certificates build-base
 ENV GOPATH /go
 RUN go version
 
-WORKDIR /go/src/smarter-device-management
+WORKDIR /arm.com/smarter-device-management
 COPY . .
 
 RUN echo $PATH;export CGO_LDFLAGS_ALLOW='-Wl,--unresolved-symbols=ignore-in-object-files' && \
-    go install -ldflags="-s -w" -v smarter-device-management
+    go mod init arm.com/smarter-device-management && go mod tidy && go mod vendor && \
+    go build -ldflags="-s -w" .
 
 FROM alpine
 
@@ -18,6 +19,6 @@ RUN apk update && apk upgrade
 WORKDIR /root
 
 COPY conf.yaml /root/config/conf.yaml
-COPY --from=build /go/bin/smarter-device-management /usr/bin/smarter-device-management
+COPY --from=build /arm.com/smarter-device-management/smarter-device-management /usr/bin/smarter-device-management
 
-CMD ["smarter-device-management","-logtostderr=true","-v=0"]
+CMD ["smarter-device-management","-logtostderr=true","-v=0"]

README.md

@@ -17,6 +17,10 @@ The smarter-device-manager starts by reading a YAML configuration file. This con
   nummaxdevices: 10
 ```
 
+Devices in subdirectories have the slash replaced with underscore in the
+resource name, due to kubernetes naming restrictions: e.g. `/dev/net/tun`
+becomes `smarter-devices/net_tun`.
+
 The default config file provided will enable most of the devices available on a Raspberry Pi (vers 1-4) or equivalent boards. I2C, SPI, video devices, sound and others would be enabled. The config file can be replaced using a configmap to enable or disable access to different devices, like accelerators, GPUs, etc.
 
 The node will show the devices it recognizes as resources in the node object in Kubernetes. The example below shows a raspberry PI.
@@ -147,3 +151,6 @@ The following command should show the node resources in a similar form as shown
 kubectl describe node pike5
 ```
 
+## k3s 
+
+K3s < 1.18 stores the plugin interface in a different directory than k8s and so it needs a different yaml file to enable smarter-device-manager to communicate correctly with k3s agent. So use the smart-device-manager-k3s yaml files on this reposistor for k3s < 1.18.

argus.go

@@ -4,7 +4,7 @@ package main
 
 import (
 	"github.com/golang/glog"
-	pluginapi "k8s.io/kubernetes/pkg/kubelet/apis/deviceplugin/v1beta1"
+        pluginapi "k8s.io/kubelet/pkg/apis/deviceplugin/v1beta1"
 )
 
 func check(err error) {

compile.sh

@@ -5,6 +5,12 @@
 function printHelp() {
                 echo $(basename $0)" options:";
                 echo "    -A <Architectures to use> # Compiling to ${ARCHS} now, examples: linux/amd64,linux/arm/v7,linux/arm/v6,linux/arm64"
+                if [ ${FLAG_NOCACHE} -gt 0 ]
+                then
+                        echo "    -C # Do not use cache"
+                else 
+                        echo "    -C # Use cache" 
+                fi
                 if [ ${FLAG_UPLOADIMAGES} -gt 0 ]
                 then
                         echo "    -U # Do not upload images - the default is upload the images to the registry"
@@ -43,8 +49,9 @@ FLAG_UPLOADMANIFEST=1
 ADDITIONAL_TAG=""
 ADDITIONAL_IMAGE_NAME=""
 PUSH_OPTION=""
+FLAG_NOCACHE=0
 
-while getopts hA:B:MST:U name
+while getopts hA:B:MST:UC name
 do
         case $name in
         h)
@@ -52,6 +59,10 @@ do
                 exit 0;;
         A)
                 ARCHS="$OPTARG";;
+        C)
+                [ ${FLAG_NOCACHE} -gt 0 ] && FLAG_NOCACHE=0;
+                [ ${FLAG_NOCACHE} -eq 0 ] && FLAG_NOCACHE=1;
+                ;;
         U)
                 [ ${FLAG_UPLOADIMAGES} -gt 0 ] && FLAG_UPLOADIMAGES=0;
                 [ ${FLAG_UPLOADIMAGES} -eq 0 ] && FLAG_UPLOADIMAGES=1;
@@ -93,6 +104,13 @@ EOF
         fi
 fi
 
+if [ $FLAG_NOCACHE -gt 0 ]
+then
+        CACHE_OPTION="--no-cache"
+else
+        CACHE_OPTION="" 
+fi
+        
 if [ $FLAG_UPLOADIMAGES -gt 0 ]
 then
         PUSH_OPTION="--push"
@@ -100,6 +118,6 @@ else
         PUSH_OPTION="--load"
 fi
         
-docker buildx build  -t "${REPOSITORY_NAME}${IMAGE_NAME}${ADDITIONAL_IMAGE_NAME}:${BUILD_TAG}" --platform=${ARCHS} ${PUSH_OPTION} .
+docker buildx build ${CACHE_OPTION}  -t "${REPOSITORY_NAME}${IMAGE_NAME}${ADDITIONAL_IMAGE_NAME}:${BUILD_TAG}" --platform=${ARCHS} ${PUSH_OPTION} .
 
 exit 0

main.go

@@ -14,7 +14,7 @@ import (
 	"github.com/fsnotify/fsnotify"
 	"github.com/golang/glog"
 	"gopkg.in/yaml.v2"
-	pluginapi "k8s.io/kubernetes/pkg/kubelet/apis/deviceplugin/v1beta1"
+        pluginapi "k8s.io/kubelet/pkg/apis/deviceplugin/v1beta1"
 )
 
 var confFileName string
@@ -56,17 +56,42 @@ func init() {
 }
 
 func readDevDirectory(dirToList string) (files []string, err error) {
-	f, err := os.Open(dirToList)
-	if err != nil {
-		return nil, err
-	}
-	files, err = f.Readdirnames(-1)
-	f.Close()
-	if err != nil {
-		return nil, err
-	}
+        var foundFiles []string
 
-	return files, nil
+        fType, err := os.Stat(dirToList)
+        if err != nil {
+                return nil, err
+        }
+
+        if !fType.IsDir()  {
+                return nil, nil
+        }
+
+        f, err := os.Open(dirToList)
+        if err != nil {
+                return nil, err
+        }
+        files, err = f.Readdirnames(-1)
+        if err != nil {
+                f.Close()
+                return nil, err
+        }
+        f.Close()
+        for _, subDir := range files {
+                foundFiles = append(foundFiles, subDir)
+                filesDir, err := readDevDirectory(dirToList+"/"+subDir)
+                if err == nil {
+                        for _, fileName := range filesDir {
+                                foundFiles = append(foundFiles, subDir+"/"+fileName)
+                        }
+                }
+        }
+
+        return foundFiles, nil
+}
+
+func sanitizeName(path string) string {
+        return strings.Replace(path, "/", "_" ,-1)
 }
 
 func findDevicesPattern(listDevices []string, pattern string) ([]string,error) {
@@ -151,9 +176,10 @@ func main() {
                         if len(foundDevices) > 0 {
                                 for _, deviceToCreate := range foundDevices {
                                         var newDevice DeviceInstance
+                                        deviceSafeName := sanitizeName(deviceToCreate)
                                         newDevice.deviceType = deviceFileType
-                                        newDevice.deviceName = "smarter-devices/" + deviceToCreate
-                                        newDevice.socketName = pluginapi.DevicePluginPath + "smarter-" + deviceToCreate + ".sock"
+                                        newDevice.deviceName = "smarter-devices/" + deviceSafeName
+                                        newDevice.socketName = pluginapi.DevicePluginPath + "smarter-" + deviceSafeName + ".sock"
                                         newDevice.deviceFile = "/dev/" + deviceToCreate
                                         newDevice.numDevices = deviceToTest.NumMaxDevices
                                         listDevicesAvailable = append(listDevicesAvailable, newDevice)
@@ -193,17 +219,17 @@ L:
 			}
 
 			var err error
-			for _, devicesInUse := range listDevicesAvailable {
-                                switch devicesInUse.deviceType {
+			for id, _ := range listDevicesAvailable {
+                                switch listDevicesAvailable[id].deviceType {
                                 case deviceFileType :
-                                        devicesInUse.devicePluginSmarter = NewSmarterDevicePlugin(devicesInUse.numDevices, devicesInUse.deviceFile, devicesInUse.deviceName, devicesInUse.socketName)
-                                        if err = devicesInUse.devicePluginSmarter.Serve(); err != nil {
+                                        listDevicesAvailable[id].devicePluginSmarter = NewSmarterDevicePlugin(listDevicesAvailable[id].numDevices, listDevicesAvailable[id].deviceFile, listDevicesAvailable[id].deviceName, listDevicesAvailable[id].socketName)
+                                        if err = listDevicesAvailable[id].devicePluginSmarter.Serve(); err != nil {
                                                 glog.V(0).Info("Could not contact Kubelet, retrying. Did you enable the device plugin feature gate?")
                                                 break
                                         }
                                 case nvidiaSysType :
-                                        devicesInUse.devicePluginNvidia = NewNvidiaDevicePlugin(devicesInUse.numDevices, devicesInUse.deviceName,"NVIDIA_VISIBLE_DEVICES", devicesInUse.socketName, devicesInUse.deviceId)
-                                        if err = devicesInUse.devicePluginNvidia.Serve(); err != nil {
+                                        listDevicesAvailable[id].devicePluginNvidia = NewNvidiaDevicePlugin(listDevicesAvailable[id].numDevices, listDevicesAvailable[id].deviceName,"NVIDIA_VISIBLE_DEVICES", listDevicesAvailable[id].socketName, listDevicesAvailable[id].deviceId)
+                                        if err = listDevicesAvailable[id].devicePluginNvidia.Serve(); err != nil {
                                                 glog.V(0).Info("Could not contact Kubelet, retrying. Did you enable the device plugin feature gate?")
                                                 break
                                         }
@@ -234,13 +260,18 @@ L:
 			default:
 				glog.V(0).Infof("Received signal \"%v\", shutting down.", s)
 				for _, devicesInUse := range listDevicesAvailable {
+                                        glog.V(0).Info("Stopping device ", devicesInUse.deviceName)
                                         switch devicesInUse.deviceType {
                                         case deviceFileType :
+				                glog.V(0).Info("Smarter device type")
                                                 if devicesInUse.devicePluginSmarter != nil {
+				                        glog.V(0).Info("Stopping device")
                                                         devicesInUse.devicePluginSmarter.Stop()
                                                 }
                                         case nvidiaSysType :
+				                glog.V(0).Info("Nvidia device type")
                                                 if devicesInUse.devicePluginNvidia != nil {
+				                        glog.V(0).Info("Stopping device")
                                                         devicesInUse.devicePluginNvidia.Stop()
                                                 }
                                         }

nvidia-server.go

@@ -13,7 +13,7 @@ import (
 	"github.com/golang/glog"
 	"golang.org/x/net/context"
 	"google.golang.org/grpc"
-	pluginapi "k8s.io/kubernetes/pkg/kubelet/apis/deviceplugin/v1beta1"
+        pluginapi "k8s.io/kubelet/pkg/apis/deviceplugin/v1beta1"
 )
 
 var passDeviceSpecs = flag.Bool("pass-device-specs", false, "pass the list of DeviceSpecs to the kubelet on Allocate()")
@@ -182,6 +182,10 @@ func (m *NvidiaDevicePlugin) PreStartContainer(context.Context, *pluginapi.PreSt
 	return &pluginapi.PreStartContainerResponse{}, nil
 }
 
+func (m *NvidiaDevicePlugin) GetPreferredAllocation(context.Context, *pluginapi.PreferredAllocationRequest) (*pluginapi.PreferredAllocationResponse, error) {
+	return &pluginapi.PreferredAllocationResponse{}, nil
+}
+
 func (m *NvidiaDevicePlugin) cleanup() error {
 	if err := os.Remove(m.socket); err != nil && !os.IsNotExist(err) {
 		return err

server.go

@@ -13,7 +13,7 @@ import (
 	"github.com/golang/glog"
 	"golang.org/x/net/context"
 	"google.golang.org/grpc"
-	pluginapi "k8s.io/kubernetes/pkg/kubelet/apis/deviceplugin/v1beta1"
+        pluginapi "k8s.io/kubelet/pkg/apis/deviceplugin/v1beta1"
 )
 
 const (
@@ -94,6 +94,7 @@ func (m *SmarterDevicePlugin) Start() error {
 
 // Stop the gRPC server
 func (m *SmarterDevicePlugin) Stop() error {
+	glog.V(0).Infof("Stopping server with socket ",m.socket)
 	if m.server == nil {
 		return nil
 	}
@@ -101,6 +102,7 @@ func (m *SmarterDevicePlugin) Stop() error {
 	m.server.Stop()
 	m.server = nil
 	close(m.stop)
+	glog.V(0).Info("Server stopped with socket ",m.socket)
 
 	return m.cleanup()
 }
@@ -178,7 +180,12 @@ func (m *SmarterDevicePlugin) PreStartContainer(context.Context, *pluginapi.PreS
 	return &pluginapi.PreStartContainerResponse{}, nil
 }
 
+func (m *SmarterDevicePlugin) GetPreferredAllocation(context.Context, *pluginapi.PreferredAllocationRequest) (*pluginapi.PreferredAllocationResponse, error) {
+	return &pluginapi.PreferredAllocationResponse{}, nil
+}
+
 func (m *SmarterDevicePlugin) cleanup() error {
+	glog.V(0).Info("Removing file ",m.socket)
 	if err := os.Remove(m.socket); err != nil && !os.IsNotExist(err) {
 		return err
 	}

smarter-device-management-pod-k3s-test-xavier.yaml

@@ -15,7 +15,7 @@ spec:
   nodeName: smarter-jetson-xavier-4bcc2584
   containers:
   - name: smarter-device-manager
-    image: registry.gitlab.com/arm-research/smarter/smarter-device-manager/smarter-device-manager:IMAGE_ID
+    image: registry.gitlab.com/arm-research/smarter/smarter-device-manager:v1.20.6
     imagePullPolicy: IfNotPresent
     securityContext:
       allowPrivilegeEscalation: false

smarter-device-management-pod-k3s.yaml

@@ -15,7 +15,7 @@ spec:
   nodeName: <replace with node to run>
   containers:
   - name: smarter-device-manager
-    image: registry.gitlab.com/arm-research/smarter/smarter-device-manager/smarter-device-manager:IMAGE_ID
+    image: registry.gitlab.com/arm-research/smarter/smarter-device-manager:v1.20.6
     imagePullPolicy: IfNotPresent
     securityContext:
       allowPrivilegeEscalation: false

smarter-device-management-pod-k8s.yaml

@@ -1,43 +0,0 @@
-apiVersion: v1
-kind: Pod
-metadata:
-  name: smarter-device-management
-  namespace: default
-spec:
-  # Mark this pod as a critical add-on; when enabled, the critical add-on
-  # scheduler reserves resources for critical add-on pods so that they can
-  # be rescheduled after a failure.
-  # See https://kubernetes.io/docs/tasks/administer-cluster/guaranteed-scheduling-critical-addon-pods/
-  priorityClassName: "system-node-critical"
-  hostNetwork: true
-  dnsPolicy: ClusterFirstWithHostNet
-  hostname: smarter-device-management
-  nodeName: <replace with node to run>
-  containers:
-  - name: smarter-device-manager
-    image: registry.gitlab.com/arm-research/smarter/smarter-device-manager/smarter-device-manager:IMAGE_ID
-    imagePullPolicy: IfNotPresent
-    securityContext:
-      allowPrivilegeEscalation: false
-      capabilities:
-        drop: ["ALL"]
-    resources:
-      limits:
-        cpu: 100m
-        memory: 10Mi
-      requests:
-        cpu: 10m
-        memory: 10Mi
-    volumeMounts:
-      - name: device-plugin
-        mountPath: /var/lib/kubelet/device-plugins
-      - name: dev-dir
-        mountPath: /dev
-  volumes:
-    - name: device-plugin
-      hostPath:
-        path: /var/lib/kubelet/device-plugins
-    - name: dev-dir
-      hostPath:
-            path: /dev
-      terminationGracePeriodSeconds: 30

smarter-device-management-pod.yaml

@@ -12,10 +12,10 @@ spec:
   hostNetwork: true
   dnsPolicy: ClusterFirstWithHostNet
   hostname: smarter-device-management
-  nodeName: smarter-jetson-xavier-4bcc2584
+  nodeName: <replace with node to run>
   containers:
   - name: smarter-device-manager
-    image: registry.gitlab.com/arm-research/smarter/smarter-device-manager/smarter-device-manager:IMAGE_ID
+    image: registry.gitlab.com/arm-research/smarter/smarter-device-manager:v1.20.6
     imagePullPolicy: IfNotPresent
     securityContext:
       allowPrivilegeEscalation: false
@@ -38,7 +38,7 @@ spec:
   volumes:
     - name: device-plugin
       hostPath:
-        path: /var/lib/rancher/k3s/agent/kubelet/device-plugins
+        path: /var/lib/kubelet/device-plugins
     - name: dev-dir
       hostPath:
             path: /dev

smarter-device-manager-configmap-rpi.yaml

@@ -22,3 +22,11 @@ data:
           nummaxdevices: 20
         - devicematch: ^vcsm.*$
           nummaxdevices: 20
+        - devicematch: ^ttyUSB[0-9]*$
+          nummaxdevices: 1
+        - devicematch: ^ttyACM[0-9]*$
+          nummaxdevices: 1
+        - devicematch: ^ttyTHS[0-9]*$
+          nummaxdevices: 1
+        - devicematch: ^ttyS[0-9]*$
+          nummaxdevices: 1

smarter-device-manager-configmap-xavier.yaml

@@ -22,5 +22,13 @@ data:
           nummaxdevices: 20
         - devicematch: ^vcsm.*$
           nummaxdevices: 20
+        - devicematch: ^ttyUSB[0-9]*$
+          nummaxdevices: 1
+        - devicematch: ^ttyACM[0-9]*$
+          nummaxdevices: 1
+        - devicematch: ^ttyTHS[0-9]*$
+          nummaxdevices: 1
+        - devicematch: ^ttyS[0-9]*$
+          nummaxdevices: 1
         - devicematch: nvidia-gpu
           nummaxdevices: 20

smarter-device-manager-ds-k3s.yaml

@@ -34,7 +34,7 @@ spec:
       dnsPolicy: ClusterFirstWithHostNet
       containers:
       - name: smarter-device-manager
-        image: registry.gitlab.com/arm-research/smarter/smarter-device-manager/smarter-device-manager:IMAGE_ID
+        image: registry.gitlab.com/arm-research/smarter/smarter-device-manager:v1.20.6
         imagePullPolicy: IfNotPresent
         securityContext:
           allowPrivilegeEscalation: false

smarter-device-manager-ds-with-configmap-rpi-k3s.yaml

@@ -34,7 +34,7 @@ spec:
       dnsPolicy: ClusterFirstWithHostNet
       containers:
       - name: smarter-device-manager
-        image: registry.gitlab.com/arm-research/smarter/smarter-device-manager/smarter-device-manager:IMAGE_ID
+        image: registry.gitlab.com/arm-research/smarter/smarter-device-manager:v1.20.6
         imagePullPolicy: IfNotPresent
         securityContext:
           allowPrivilegeEscalation: false

smarter-device-manager-ds-with-configmap-rpi.yaml

@@ -32,11 +32,9 @@ spec:
       hostname: smarter-device-management
       hostNetwork: true
       dnsPolicy: ClusterFirstWithHostNet
-      imagePullSecrets:
-      - name: k8sedgeregcred
       containers:
       - name: smarter-device-manager
-        image: registry.gitlab.com/arm-research/smarter/smarter-device-manager/smarter-device-manager:IMAGE_ID
+        image: registry.gitlab.com/arm-research/smarter/smarter-device-manager:v1.20.6
         imagePullPolicy: IfNotPresent
         securityContext:
           allowPrivilegeEscalation: false

smarter-device-manager-ds-with-configmap-xavier-k3s.yaml

@@ -32,11 +32,9 @@ spec:
       hostname: smarter-device-management
       hostNetwork: true
       dnsPolicy: ClusterFirstWithHostNet
-      imagePullSecrets:
-      - name: k8sedgeregcred
       containers:
       - name: smarter-device-manager
-        image: registry.gitlab.com/arm-research/smarter/smarter-device-manager/smarter-device-manager:IMAGE_ID
+        image: registry.gitlab.com/arm-research/smarter/smarter-device-manager:v1.20.6
         imagePullPolicy: IfNotPresent
         securityContext:
           allowPrivilegeEscalation: false

smarter-device-manager-ds-with-configmap-xavier.yaml

@@ -32,11 +32,9 @@ spec:
       hostname: smarter-device-management
       hostNetwork: true
       dnsPolicy: ClusterFirstWithHostNet
-      imagePullSecrets:
-      - name: k8sedgeregcred
       containers:
       - name: smarter-device-manager
-        image: registry.gitlab.com/arm-research/smarter/smarter-device-manager/smarter-device-manager:IMAGE_ID
+        image: registry.gitlab.com/arm-research/smarter/smarter-device-manager:v1.20.6
         imagePullPolicy: IfNotPresent
         securityContext:
           allowPrivilegeEscalation: false

smarter-device-manager-ds.yaml

@@ -32,11 +32,9 @@ spec:
       hostname: smarter-device-management
       hostNetwork: true
       dnsPolicy: ClusterFirstWithHostNet
-      imagePullSecrets:
-      - name: k8sedgeregcred
       containers:
       - name: smarter-device-manager
-        image: registry.gitlab.com/arm-research/smarter/smarter-device-manager/smarter-device-manager:IMAGE_ID
+        image: registry.gitlab.com/arm-research/smarter/smarter-device-manager:v1.20.6
         imagePullPolicy: IfNotPresent
         securityContext:
           allowPrivilegeEscalation: false