package rclone import ( "fmt" "io/ioutil" "os" "os/exec" "strings" v1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/klog" "github.com/container-storage-interface/spec/lib/go/csi" "golang.org/x/net/context" "google.golang.org/grpc/codes" "google.golang.org/grpc/status" "k8s.io/client-go/tools/clientcmd" "k8s.io/kubernetes/pkg/util/mount" "k8s.io/kubernetes/pkg/volume/util" csicommon "github.com/kubernetes-csi/drivers/pkg/csi-common" ) type nodeServer struct { *csicommon.DefaultNodeServer mounter *mount.SafeFormatAndMount } type mountPoint struct { VolumeId string MountPath string } func (ns *nodeServer) NodePublishVolume(ctx context.Context, req *csi.NodePublishVolumeRequest) (*csi.NodePublishVolumeResponse, error) { klog.Infof("NodePublishVolume: called with args %+v", *req) targetPath := req.GetTargetPath() notMnt, err := mount.New("").IsLikelyNotMountPoint(targetPath) if err != nil { if os.IsNotExist(err) { if err := os.MkdirAll(targetPath, 0750); err != nil { return nil, status.Error(codes.Internal, err.Error()) } notMnt = true } else { return nil, status.Error(codes.Internal, err.Error()) } } if !notMnt { // testing original mount point, make sure the mount link is valid if _, err := ioutil.ReadDir(targetPath); err == nil { klog.Infof("already mounted to target %s", targetPath) return &csi.NodePublishVolumeResponse{}, nil } // todo: mount link is invalid, now unmount and remount later (built-in functionality) klog.Warningf("ReadDir %s failed with %v, unmount this directory", targetPath, err) ns.mounter = &mount.SafeFormatAndMount{ Interface: mount.New(""), Exec: mount.NewOsExec(), } if err := ns.mounter.Unmount(targetPath); err != nil { klog.Errorf("Unmount directory %s failed with %v", targetPath, err) return nil, err } } mountOptions := req.GetVolumeCapability().GetMount().GetMountFlags() if req.GetReadonly() { mountOptions = append(mountOptions, "ro") } remote, remotePath, configData, flags, e := extractFlags(req.GetVolumeContext()) if e != nil { klog.Warningf("storage parameter error: %s", e) return nil, e } e = Mount(remote, remotePath, targetPath, configData, flags) if e != nil { if os.IsPermission(e) { return nil, status.Error(codes.PermissionDenied, e.Error()) } if strings.Contains(e.Error(), "invalid argument") { return nil, status.Error(codes.InvalidArgument, e.Error()) } return nil, status.Error(codes.Internal, e.Error()) } return &csi.NodePublishVolumeResponse{}, nil } // extractFlags extracts the flags from the given volumeContext // Retturns: remote, remotePath, configData, flags, error func extractFlags(volumeContext map[string]string) (string, string, string, map[string]string, error) { // Load default connection settings from secret var secret *v1.Secret if secretName, ok := volumeContext["secretName"]; ok { // Load the secret that the PV spec defines var e error secret, e = getSecret(secretName) if e != nil { // if the user explicitly requested a secret and there is an error fetching it, bail with an error return "", "", "", nil, e } } else { // use rclone-secret as the default secret if none was defined secret, _ = getSecret("rclone-secret") } // Empty argument list flags := make(map[string]string) // Secret values are default, gets merged and overriden by corresponding PV values if secret != nil && secret.Data != nil && len(secret.Data) > 0 { // Needs byte to string casting for map values for k, v := range secret.Data { flags[k] = string(v) } } else { klog.Infof("No csi-rclone connection defaults secret found.") } if len(volumeContext) > 0 { for k, v := range volumeContext { flags[k] = v } } if e := validateFlags(flags); e != nil { return "", "", "", flags, e } remote := flags["remote"] remotePath := flags["remotePath"] if remotePathSuffix, ok := flags["remotePathSuffix"]; ok { remotePath = remotePath + remotePathSuffix delete(flags, "remotePathSuffix") } configData := "" ok := false if configData, ok = flags["configData"]; ok { delete(flags, "configData") } delete(flags, "remote") delete(flags, "remotePath") delete(flags, "secretName") return remote, remotePath, configData, flags, nil } func (ns *nodeServer) NodeUnpublishVolume(ctx context.Context, req *csi.NodeUnpublishVolumeRequest) (*csi.NodeUnpublishVolumeResponse, error) { klog.Infof("NodeUnPublishVolume: called with args %+v", *req) targetPath := req.GetTargetPath() if len(targetPath) == 0 { return nil, status.Error(codes.InvalidArgument, "NodeUnpublishVolume Target Path must be provided") } m := mount.New("") notMnt, err := m.IsLikelyNotMountPoint(targetPath) if err != nil && !mount.IsCorruptedMnt(err) { return nil, status.Error(codes.Internal, err.Error()) } if notMnt && !mount.IsCorruptedMnt(err) { klog.Infof("Volume not mounted") } else { err = util.UnmountPath(req.GetTargetPath(), m) if err != nil { klog.Infof("Error while unmounting path: %s", err) // This will exit and fail the NodeUnpublishVolume making it to retry unmount on the next api schedule trigger. // Since we mount the volume with allow-non-empty now, we could skip this one too. return nil, status.Error(codes.Internal, err.Error()) } klog.Infof("Volume %s unmounted successfully", req.VolumeId) } return &csi.NodeUnpublishVolumeResponse{}, nil } func (ns *nodeServer) NodeUnstageVolume(ctx context.Context, req *csi.NodeUnstageVolumeRequest) (*csi.NodeUnstageVolumeResponse, error) { klog.Infof("NodeUnstageVolume: called with args %+v", *req) return &csi.NodeUnstageVolumeResponse{}, nil } func (ns *nodeServer) NodeStageVolume(ctx context.Context, req *csi.NodeStageVolumeRequest) (*csi.NodeStageVolumeResponse, error) { klog.Infof("NodeStageVolume: called with args %+v", *req) return &csi.NodeStageVolumeResponse{}, nil } func validateFlags(flags map[string]string) error { if _, ok := flags["remote"]; !ok { return status.Errorf(codes.InvalidArgument, "missing volume context value: remote") } if _, ok := flags["remotePath"]; !ok { return status.Errorf(codes.InvalidArgument, "missing volume context value: remotePath") } return nil } func getSecret(secretName string) (*v1.Secret, error) { clientset, e := GetK8sClient() if e != nil { return nil, status.Errorf(codes.Internal, "can not create kubernetes client: %s", e) } kubeconfig := clientcmd.NewNonInteractiveDeferredLoadingClientConfig( clientcmd.NewDefaultClientConfigLoadingRules(), &clientcmd.ConfigOverrides{}, ) namespace, _, err := kubeconfig.Namespace() if err != nil { return nil, status.Errorf(codes.Internal, "can't get current namespace, error %s", secretName, err) } klog.Infof("Loading csi-rclone connection defaults from secret %s/%s", namespace, secretName) secret, e := clientset.CoreV1(). Secrets(namespace). Get(secretName, metav1.GetOptions{}) if e != nil { return nil, status.Errorf(codes.Internal, "can't load csi-rclone settings from secret %s: %s", secretName, e) } return secret, nil } // Mount routine. func Mount(remote string, remotePath string, targetPath string, configData string, flags map[string]string) error { mountCmd := "rclone" mountArgs := []string{} defaultFlags := map[string]string{} defaultFlags["cache-info-age"] = "72h" defaultFlags["cache-chunk-clean-interval"] = "15m" defaultFlags["dir-cache-time"] = "5s" defaultFlags["vfs-cache-mode"] = "writes" defaultFlags["allow-non-empty"] = "true" defaultFlags["allow-other"] = "true" remoteWithPath := fmt.Sprintf(":%s:%s", remote, remotePath) if strings.Contains(configData, "["+remote+"]") { remoteWithPath = fmt.Sprintf("%s:%s", remote, remotePath) klog.Infof("remote %s found in configData, remoteWithPath set to %s", remote, remoteWithPath) } // rclone mount remote:path /path/to/mountpoint [flags] mountArgs = append( mountArgs, "mount", remoteWithPath, targetPath, "--daemon", ) // If a custom flag configData is defined, // create a temporary file, fill it with configData content, // and run rclone with --config flag if configData != "" { configFile, err := ioutil.TempFile("", "rclone.conf") if err != nil { return err } // Normally, a defer os.Remove(configFile.Name()) should be placed here. // However, due to a rclone mount --daemon flag, rclone forks and creates a race condition // with this nodeplugin proceess. As a result, the config file gets deleted // before it's reread by a forked process. if _, err := configFile.Write([]byte(configData)); err != nil { return err } if err := configFile.Close(); err != nil { return err } mountArgs = append(mountArgs, "--config", configFile.Name()) } // Add default flags for k, v := range defaultFlags { // Exclude overriden flags if _, ok := flags[k]; !ok { mountArgs = append(mountArgs, fmt.Sprintf("--%s=%s", k, v)) } } // Add user supplied flags for k, v := range flags { mountArgs = append(mountArgs, fmt.Sprintf("--%s=%s", k, v)) } // create target, os.Mkdirall is noop if it exists err := os.MkdirAll(targetPath, 0750) if err != nil { return err } klog.Infof("executing mount command cmd=%s, remote=%s, targetpath=%s", mountCmd, remoteWithPath, targetPath) out, err := exec.Command(mountCmd, mountArgs...).CombinedOutput() if err != nil { return fmt.Errorf("mounting failed: %v cmd: '%s' remote: '%s' targetpath: %s output: %q", err, mountCmd, remoteWithPath, targetPath, string(out)) } return nil }