Problems with file paths

General Usage
1

Hello !

I am using a Python script that calls a shell script, which in turn executes a Julia script.

Here is the python script :

import os
import subprocess

BUFFERSIZE = 72000
def julia_spike_filter(projectPath, folderCode, windowSize = 0.036, singleSpike=False):
    # Launch an extraction of the spikes in Julia:
    if singleSpike:
        test1 = os.path.isfile((os.path.join(projectPath,"dataset","dataset_singleSpike.tfrec")))
    else:
        test1 = os.path.isfile((os.path.join(projectPath, "dataset", "dataset_stride"+str(round(windowSize*1000))+".tfrec")))
    if not test1 :
        if not os.path.exists(os.path.join(projectPath,'behavResources.mat')):
            raise ValueError('the behavior file does not exist :' + os.path.join(projectPath,'behavResources.mat'))
        if not os.path.exists(projectPath):
            raise ValueError('the dat file does not exist :' + projectPath)
        codepath = os.path.join(folderCode,"importData/juliaData/")
        if singleSpike:
            subprocess.run([codepath + "executeFilter_singleSpike.sh",
                            projectPath,
                            projectPath,
                            projectPath,
                            os.path.join(projectPath,"behavResources.mat"),
                            os.path.join(projectPath,"spikeData_fromJulia.csv"),
                            os.path.join(projectPath,"dataset","dataset_singleSpike.tfrec"),
                            os.path.join(projectPath, "dataset", "datasetSleep_singleSpike.tfrec"),
                            str(BUFFERSIZE),
                            str(windowSize)])
        else:
            subprocess.run([os.path.join(codepath, "executeFilter_stride.sh"),
                            projectPath,
                            projectPath,
                            projectPath,
                            os.path.join(projectPath,"behavResources.mat"),
                            os.path.join(projectPath,"spikeData_fromJulia.csv"),
                            os.path.join(projectPath,"dataset","dataset_stride"+str(round(windowSize*1000))+".tfrec"),
                            os.path.join(projectPath, "dataset", "datasetSleep_stride"+str(round(windowSize*1000))+".tfrec"),
                            str(BUFFERSIZE),
                            str(windowSize),
                            str(0.036)]) #the striding is 36ms based...

julia_spike_filter("/media/nas6/ProjetERC2/Mouse-K199/20210408/_Concatenated/TEST", "/home/mobs/Dropbox/Mobs_member/Basile/Code/full_neuroEncoder/neuroEncoders-master")

Here is the shell script :

#!/bin/bash

echo "Starting Spike extraction using the julia language"
cd $1
pwd
/home/mobs/julia-1.6.1/bin/julia spikeFilter_withStride.jl --projectDir=$1 $2 $3 $4 $5 $6 $7 $8 $9 ${10} ${11}

And here is the beginning of the Julia script (It’s very long and the problem seems to come from the first 100 lines) :

using Pkg
Pkg.activate(".")
using CSV
using MAT
using ProgressBars
using HDF5
using LightXML
using Mmap
using Distributions
using Interpolations
using DataFrames
using TFRecord

# A small julia script to extract very efficiently
# spikes times and align them with a nnBehavior matrix.
# The results is stored in a CSV folder.

# The script should be launched with the following arguments, in order:
#   the path to the xml file
#   the path to the dat file
#   the path to the nnBehavior.mat file
#   the path at which the output csv file will be written
# 	the path to the dataset where we will write the awake spike
# 	the path to the dataset where we will write the sleeping spike
# e.g:
# xmlPath = "~/dataTest/Mouse-M1199-1304/continuous.xml"
# datPath = "~/dataTest/Mouse-M1199-1304/continuous.dat"
# behavePath = "~/dataTest/Mouse-M1199-1304/nnBehavior.mat"
# fileName = "~/dataTest/Mouse-M1199-1304/test_spikeJulia.csv"
#
xmlPath = ARGS[2]
datPath = ARGS[3]
behavePath = ARGS[4]
fileName = ARGS[5]
thresholdsFileName = replace(fileName,"spikeData_fromJulia"=>"thresholds_Julia")
datasetName = ARGS[6]
datasetNameSleep = ARGS[7]
BUFFERSIZE = parse(Int64,ARGS[8])
WINDOWSIZE = parse(Float32,ARGS[9])
WINDOWSTRIDE = parse(Float32,ARGS[10])


print(xmlPath)

using CodecZlib
using BufferedStreams
#we modify the TFrecord so it Effectively appends
function TFRecord.write(s::AbstractString, x;compression=nothing, bufsize=1024*1024)
    io = BufferedOutputStream(open(s, "a"), bufsize)
    if compression == :gzip
        io = GzipCompressorStream(io)
    elseif compression == :zlib
        io = ZlibCompressorStream(io)
    else
        isnothing(compression) || throw(ArgumentError("unsupported compression method: $compression"))
    end
    TFRecord.write(io, x)
    close(io)
end


function isInEpochs(time,epochs)
	# for an epochs array in the format [[start,end,start,end,....]]
	# verify if time is in any of these epochs:
	if size(epochs,1)>0
		return map(t->sum((t.>=epochs[1:2:end-1,1]).*(t.<epochs[2:2:end,1]))>0,time)
	else
		return map(t->false,time)
	end
end


function extract_spike_with_buffer(xmlPath,datPath,behavePath,fileName,datasetName,datasetNameSleep,BUFFERSIZE,WINDOWSIZE)
	#Read the xml file:
	xdoc = parse_file(xmlPath)
	xroot = root(xdoc)
	acquiSystem =  xroot["acquisitionSystem"]
	nbits = parse(Float16,content(acquiSystem[1]["nBits"][1]))
	Nchannel = parse(Float16,content(acquiSystem[1]["nChannels"][1]))
	samplingRate = parse(Float16,content(acquiSystem[1]["samplingRate"][1]))
	spd = xroot["spikeDetection"]
	groupList = spd[1]["channelGroups"][1]["group"]
	pint =s-> parse(Int64,s)
	#We need to add 1 to all channels because they are indexed starting at 0!!
	list_channels = map(s->pint.(map(c->content(c),s["channels"][1]["channel"])).+1,groupList)

    println("Number of bits: ",nbits)
    println("Number of channel: ",Nchannel)
    println("Sampling rate: ",samplingRate)
    println("List of channels for each group: " ,list_channels)
	println("Window size: " ,string(WINDOWSIZE*1000))

	file = open(datPath)
	isreadable(file)
	#memory-map the file:
	mmapFile = Mmap.mmap(file,Matrix{Int16},(Int(Nchannel),Int(filesize(file)/(2.0*Nchannel))))
	mmapFile = transpose(mmapFile)
	channel_focus = vcat(list_channels...)

	# first we will associate each spike to a measurement of environmental feature
	# and the time at which it was measured
	behaveMat = h5open(behavePath)
	position_time = behaveMat["behavior"]["position_time"][:,:]
	speed = behaveMat["behavior"]["speed"][:,:]
	positions = behaveMat["behavior"]["positions"][:,:]
	sleepPeriods =[]
	try
		# sleepPeriods = vcat(behaveMat["behavior"]["sleepPeriods"][:]...)
		# sleepPeriods = reshape(sleepPeriods,(size(sleepPeriods,1),1))
		sleepPeriods = behaveMat["behavior"]["sleepPeriods"][:,:]
		sleepPeriods = reshape(sleepPeriods,(size(sleepPeriods,2),1))
		println("Detected sleep periods in nnBehavior.mat")
		println(sleepPeriods)
	catch
		println("No sleep periods detected in nnBehavior.mat, saving all spikes in the position_time limit in the same dataset")
		sleepPeriods =[]
	end
	maxpos = maximum(positions)

	nodes = (position_time[:,1],)
	nodes_index = float.(1:1:size(position_time,1))
	itp = interpolate(nodes, nodes_index, (Gridded(Constant())))

	nGroups = size(list_channels,1)

	print("subtracting the low pass (fc=350Hz) filtered voltage to itself ")

	buffer_mmap = zeros(Float32,BUFFERSIZE+15+16,size(mmapFile,2))
	#15 at the beginning to complete early spikes of the buffer
	#16 at the end to complete final spike of the buffer
	state = zeros(Float32,size(channel_focus,1))
	old_possibleSpike_sum = map(id->zeros(Int64,14),1:1:nGroups)
###etc

So the problem here is that when I run the python script I get the following error message :

ERROR: SystemError: opening file “/media/nas6/ProjetERC2/Mouse-K199/20210408/_Concatenated/TEST/spikeFilter_withStride.jl”: No such file or directory
Stacktrace:
[1] systemerror(p::String, errno::Int32; extrainfo::Nothing)
@ Base ./error.jl:168
[2] #systemerror#62
@ ./error.jl:167 [inlined]
[3] systemerror
@ ./error.jl:167 [inlined]
[4] open(fname::String; lock::Bool, read::Nothing, write::Nothing, create::Nothing, truncate::Nothing, append::Nothing)
@ Base ./iostream.jl:293
[5] open
@ ./iostream.jl:282 [inlined]
[6] open(f::Base.var"#326#327"{String}, args::String; kwargs::Base.Iterators.Pairs{Union{}, Union{}, Tuple{}, NamedTuple{(), Tuple{}}})
@ Base ./io.jl:328
[7] open
@ ./io.jl:328 [inlined]
[8] read
@ ./io.jl:434 [inlined]
[9] _include(mapexpr::Function, mod::Module, _path::String)
@ Base ./loading.jl:1144
[10] include(mod::Module, _path::String)
@ Base ./Base.jl:386
[11] exec_options(opts::Base.JLOptions)
@ Base ./client.jl:285
[12] _start()
@ Base ./client.jl:485

And I just cannot understand why it is trying to access "/media/nas6/ProjetERC2/Mouse-K199/20210408/_Concatenated/TEST/spikeFilter_withStride.jl"

Indeed, the first argument being passed to the shell script is
os.path.join(codepath, "executeFilter_stride.sh")
and
codepath = os.path.join(folderCode,"importData/juliaData/")
and
folderCode="/home/mobs/Dropbox/Mobs_member/Basile/Code/full_neuroEncoder/neuroEncoders-master"

And the second argument being passed to the shell script is
projectPath="/media/nas6/ProjetERC2/Mouse-K199/20210408/_Concatenated/TEST"

So I don’t see at what point and in which script there is an occurence of anything leading to
"/media/nas6/ProjetERC2/Mouse-K199/20210408/_Concatenated/TEST/spikeFilter_withStride.jl"

By the way
"/media/nas6/ProjetERC2/Mouse-K199/20210408/_Concatenated/TEST"
is supposed to be the folder containing the .xml, .dat, and .mat file. That’s why I want it to be the ARGS[2] ARGS[3] and ARGS[4] of the shell script,as you can understand with the following part of the Julia script :

xmlPath = ARGS[2]
datPath = ARGS[3]
behavePath = ARGS[4]
2

This does not look like a problem within the julia script. It looks like it cannot find the script to begin with. Are you sure it is located in whichever folder you are cd to in the bash script?

3

My first guess is that the filename spikeFilter_withStride.jl here should be specified using the absolute filepath, otherwise the file will be assumed to be present in whatever the cwd is.

1 Like
4

OOOPS my bad what I didn’t tell you (because I myself forgot about it) is that those script are actually used by many other scripts and in fact I forgot to look at the larger picture. projectPath isn’t even supposed to be a string but rather an instance of a class called projectPath.

Anyways, forget about it, sorry to have taken your time