From 5b8e1996273572de5c36e8d1fcaec22ac89a6820 Mon Sep 17 00:00:00 2001 From: Martin Diehl Date: Tue, 9 Feb 2021 23:09:41 +0100 Subject: [PATCH 1/5] avoid errors related to CRLF (windows) file endings --- src/IO.f90 | 21 +++++++++++++++++++-- 1 file changed, 19 insertions(+), 2 deletions(-) diff --git a/src/IO.f90 b/src/IO.f90 index fd87907aa..33ab7ee97 100644 --- a/src/IO.f90 +++ b/src/IO.f90 @@ -65,8 +65,8 @@ end subroutine IO_init function IO_readlines(fileName) result(fileContent) character(len=*), intent(in) :: fileName - character(len=pStringLen), dimension(:), allocatable :: fileContent !< file content, separated per lines + character(len=pStringLen) :: line character(len=:), allocatable :: rawData integer :: & @@ -75,6 +75,7 @@ function IO_readlines(fileName) result(fileContent) l logical :: warned + rawData = IO_read(fileName) !-------------------------------------------------------------------------------------------------- @@ -112,17 +113,20 @@ end function IO_readlines !-------------------------------------------------------------------------------------------------- !> @brief Read whole file. -!> @details ensures that the string ends with a new line (expected UNIX behavior) +!> @details ensures that the string ends with a new line (expected UNIX behavior) and rejects +! windows (CRLF) line endings !-------------------------------------------------------------------------------------------------- function IO_read(fileName) result(fileContent) character(len=*), intent(in) :: fileName character(len=:), allocatable :: fileContent + integer :: & fileLength, & fileUnit, & myStat + inquire(file = fileName, size=fileLength) open(newunit=fileUnit, file=fileName, access='stream',& status='old', position='rewind', action='read',iostat=myStat) @@ -137,6 +141,8 @@ function IO_read(fileName) result(fileContent) if(myStat /= 0) call IO_error(102,ext_msg=trim(fileName)) close(fileUnit) + if(scan(fileContent,achar(13)) /= 0) call IO_error(115) + if(fileContent(fileLength:fileLength) /= IO_EOL) fileContent = fileContent//IO_EOL ! ensure EOL@EOF end function IO_read @@ -151,6 +157,7 @@ logical pure function IO_isBlank(string) integer :: posNonBlank + posNonBlank = verify(string,IO_WHITESPACE) IO_isBlank = posNonBlank == 0 .or. posNonBlank == scan(string,IO_COMMENT) @@ -170,6 +177,7 @@ pure function IO_stringPos(string) integer :: left, right + allocate(IO_stringPos(1), source=0) right = 0 @@ -249,6 +257,7 @@ pure function IO_lc(string) integer :: i,n + do i=1,len(string) n = index(UPPER,string(i:i)) if(n/=0) then @@ -271,6 +280,7 @@ function IO_rmComment(line) character(len=:), allocatable :: IO_rmComment integer :: split + split = index(line,IO_COMMENT) if (split == 0) then @@ -292,6 +302,7 @@ integer function IO_stringAsInt(string) integer :: readStatus character(len=*), parameter :: VALIDCHARS = '0123456789+- ' + valid: if (verify(string,VALIDCHARS) == 0) then read(string,*,iostat=readStatus) IO_stringAsInt if (readStatus /= 0) call IO_error(111,ext_msg=string) @@ -313,6 +324,7 @@ real(pReal) function IO_stringAsFloat(string) integer :: readStatus character(len=*), parameter :: VALIDCHARS = '0123456789eE.+- ' + valid: if (verify(string,VALIDCHARS) == 0) then read(string,*,iostat=readStatus) IO_stringAsFloat if (readStatus /= 0) call IO_error(112,ext_msg=string) @@ -331,6 +343,7 @@ logical function IO_stringAsBool(string) character(len=*), intent(in) :: string !< string for conversion to int value + if (trim(adjustl(string)) == 'True' .or. trim(adjustl(string)) == 'true') then IO_stringAsBool = .true. elseif (trim(adjustl(string)) == 'False' .or. trim(adjustl(string)) == 'false') then @@ -356,6 +369,7 @@ subroutine IO_error(error_ID,el,ip,g,instance,ext_msg) character(len=:), allocatable :: msg character(len=pStringLen) :: formatString + select case (error_ID) !-------------------------------------------------------------------------------------------------- @@ -382,6 +396,9 @@ subroutine IO_error(error_ID,el,ip,g,instance,ext_msg) msg = 'invalid character for logical:' case (114) msg = 'cannot decode base64 string:' + case (115) + msg = 'found CR. Windows file endings (CRLF) are not supported.' + !-------------------------------------------------------------------------------------------------- ! lattice error messages From ef45e856a15cb1f265b08f5c8934a8fc28ad6489 Mon Sep 17 00:00:00 2001 From: Martin Diehl Date: Wed, 10 Feb 2021 09:07:42 +0100 Subject: [PATCH 2/5] don't scan the whole file in case of proper line endings might lead to strange behavior if people randomly distribute CRs in their file. But that actually deserves to get strange behavior + Test --- PRIVATE | 2 +- src/IO.f90 | 8 ++++++-- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/PRIVATE b/PRIVATE index d90babadf..5d27d879a 160000 --- a/PRIVATE +++ b/PRIVATE @@ -1 +1 @@ -Subproject commit d90babadfb0a33afa1b793044cc4efd4d7430731 +Subproject commit 5d27d879abaeb1542e3f9f3065172be740af2899 diff --git a/src/IO.f90 b/src/IO.f90 index 33ab7ee97..36b774191 100644 --- a/src/IO.f90 +++ b/src/IO.f90 @@ -124,7 +124,9 @@ function IO_read(fileName) result(fileContent) integer :: & fileLength, & fileUnit, & - myStat + myStat, & + firstEOL + character, parameter :: CR = achar(13) inquire(file = fileName, size=fileLength) @@ -141,10 +143,12 @@ function IO_read(fileName) result(fileContent) if(myStat /= 0) call IO_error(102,ext_msg=trim(fileName)) close(fileUnit) - if(scan(fileContent,achar(13)) /= 0) call IO_error(115) if(fileContent(fileLength:fileLength) /= IO_EOL) fileContent = fileContent//IO_EOL ! ensure EOL@EOF + firstEOL = index(fileContent,IO_EOL) + if(scan(fileContent(firstEOL:firstEOL),CR) /= 0) call IO_error(115) + end function IO_read From 6895ef6b18b2cc3b3c26d0ab09c81e0115016b21 Mon Sep 17 00:00:00 2001 From: Martin Diehl Date: Wed, 10 Feb 2021 10:03:35 +0100 Subject: [PATCH 3/5] always write LF line endings small pitfall: I windows users use a filehandle that results from a call to open() without the newline option, they get still CRLF line endings --- python/damask/_colormap.py | 16 ++++++++-------- python/damask/_config.py | 2 +- python/damask/_result.py | 2 +- python/damask/_table.py | 2 +- python/damask/solver/_marc.py | 2 +- 5 files changed, 12 insertions(+), 12 deletions(-) diff --git a/python/damask/_colormap.py b/python/damask/_colormap.py index 5a22f049b..8af14e081 100644 --- a/python/damask/_colormap.py +++ b/python/damask/_colormap.py @@ -238,7 +238,7 @@ class Colormap(mpl.colors.ListedColormap): fhandle = None else: try: - fhandle = open(fname,'w') + fhandle = open(fname,'w',newline='\n') except TypeError: fhandle = fname @@ -254,7 +254,7 @@ class Colormap(mpl.colors.ListedColormap): 'RGBPoints':colors }] - with open(self.name.replace(' ','_')+'.json', 'w') if fhandle is None else fhandle as f: + with open(self.name.replace(' ','_')+'.json','w',newline='\n') if fhandle is None else fhandle as f: json.dump(out, f,indent=4) @@ -273,14 +273,14 @@ class Colormap(mpl.colors.ListedColormap): fhandle = None else: try: - fhandle = open(fname,'w') + fhandle = open(fname,'w',newline='\n') except TypeError: fhandle = fname labels = {'RGBA':4} if self.colors.shape[1] == 4 else {'RGB': 3} t = Table(self.colors,labels,f'Creator: {util.execution_stamp("Colormap")}') - with open(self.name.replace(' ','_')+'.txt', 'w') if fhandle is None else fhandle as f: + with open(self.name.replace(' ','_')+'.txt','w',newline='\n') if fhandle is None else fhandle as f: t.save(f) @@ -299,7 +299,7 @@ class Colormap(mpl.colors.ListedColormap): fhandle = None else: try: - fhandle = open(fname,'w') + fhandle = open(fname,'w',newline='\n') except TypeError: fhandle = fname # ToDo: test in GOM @@ -308,7 +308,7 @@ class Colormap(mpl.colors.ListedColormap): + f'30 NO_UNIT 1 1 64 64 64 255 1 0 0 0 0 0 0 3 0 {len(self.colors)}' \ + ' '.join([f' 0 {c[0]} {c[1]} {c[2]} 255 1' for c in reversed((self.colors*255).astype(int))]) \ + '\n' - with open(self.name.replace(' ','_')+'.legend', 'w') if fhandle is None else fhandle as f: + with open(self.name.replace(' ','_')+'.legend','w',newline='\n') if fhandle is None else fhandle as f: f.write(GOM_str) @@ -327,14 +327,14 @@ class Colormap(mpl.colors.ListedColormap): fhandle = None else: try: - fhandle = open(fname,'w') + fhandle = open(fname,'w',newline='\n') except TypeError: fhandle = fname # ToDo: test in gmsh gmsh_str = 'View.ColorTable = {\n' \ +'\n'.join([f'{c[0]},{c[1]},{c[2]},' for c in self.colors[:,:3]*255]) \ +'\n}\n' - with open(self.name.replace(' ','_')+'.msh', 'w') if fhandle is None else fhandle as f: + with open(self.name.replace(' ','_')+'.msh','w',newline='\n') if fhandle is None else fhandle as f: f.write(gmsh_str) diff --git a/python/damask/_config.py b/python/damask/_config.py index 9aa031ff0..edf3a5676 100644 --- a/python/damask/_config.py +++ b/python/damask/_config.py @@ -75,7 +75,7 @@ class Config(dict): """ try: - fhandle = open(fname,'w') + fhandle = open(fname,'w',newline='\n') except TypeError: fhandle = fname diff --git a/python/damask/_result.py b/python/damask/_result.py index 3d8368911..6f2494299 100644 --- a/python/damask/_result.py +++ b/python/damask/_result.py @@ -1287,7 +1287,7 @@ class Result: np.prod(shape))} data_items[-1].text=f'{os.path.split(self.fname)[1]}:{name}' - with open(self.fname.with_suffix('.xdmf').name,'w') as f: + with open(self.fname.with_suffix('.xdmf').name,'w',newline='\n') as f: f.write(xml.dom.minidom.parseString(ET.tostring(xdmf).decode()).toprettyxml()) diff --git a/python/damask/_table.py b/python/damask/_table.py index 78a8a276e..e9acb371d 100644 --- a/python/damask/_table.py +++ b/python/damask/_table.py @@ -380,7 +380,7 @@ class Table: [f'# {comment}' for comment in self.comments] try: - fhandle = open(fname,'w') + fhandle = open(fname,'w',newline='\n') except TypeError: fhandle = fname diff --git a/python/damask/solver/_marc.py b/python/damask/solver/_marc.py index d4aadb7ff..f6c81da9f 100644 --- a/python/damask/solver/_marc.py +++ b/python/damask/solver/_marc.py @@ -66,7 +66,7 @@ class Marc: if logfile is not None: try: - f = open(logfile,'w+') + f = open(logfile,'w+',newline='\n') except TypeError: f = logfile else: From 4e31862f0ff9cae6c89687bdad26ff660dd216d4 Mon Sep 17 00:00:00 2001 From: Martin Diehl Date: Wed, 10 Feb 2021 18:35:13 +0100 Subject: [PATCH 4/5] avoid repetition --- python/damask/_colormap.py | 75 ++++++++++++++++++-------------------- python/damask/_table.py | 2 +- 2 files changed, 36 insertions(+), 41 deletions(-) diff --git a/python/damask/_colormap.py b/python/damask/_colormap.py index 8af14e081..7e8860dae 100644 --- a/python/damask/_colormap.py +++ b/python/damask/_colormap.py @@ -223,25 +223,46 @@ class Colormap(mpl.colors.ListedColormap): return Colormap(np.array(rev.colors),rev.name[:-4] if rev.name.endswith('_r_r') else rev.name) - def save_paraview(self,fname=None): + def _get_file_handle(self,fname,extension): """ - Save as JSON file for use in Paraview. + Provide file handle. Parameters ---------- - fname : file, str, or pathlib.Path, optional. - Filename to store results. If not given, the filename will - consist of the name of the colormap and extension '.json'. + fname : file, str, pathlib.Path, or None + Filename or filehandle, will be name of the colormap+extension if None. + + extension: str + Extension of the filename. + + Returns + ------- + f + File handle """ if fname is None: - fhandle = None + fhandle = open(self.name.replace(' ','_')+'.'+extension,'w',newline='\n') else: try: fhandle = open(fname,'w',newline='\n') except TypeError: fhandle = fname + return fhandle + + + def save_paraview(self,fname=None): + """ + Save as JSON file for use in Paraview. + + Parameters + ---------- + fname : file, str, or pathlib.Path, optional + Filename to store results. If not given, the filename will + consist of the name of the colormap and extension '.json'. + + """ colors = [] for i,c in enumerate(np.round(self.colors,6).tolist()): colors+=[i]+c @@ -254,8 +275,7 @@ class Colormap(mpl.colors.ListedColormap): 'RGBPoints':colors }] - with open(self.name.replace(' ','_')+'.json','w',newline='\n') if fhandle is None else fhandle as f: - json.dump(out, f,indent=4) + json.dump(out,self._get_file_handle(fname,'json'),indent=4) def save_ASCII(self,fname=None): @@ -264,24 +284,14 @@ class Colormap(mpl.colors.ListedColormap): Parameters ---------- - fname : file, str, or pathlib.Path, optional. + fname : file, str, or pathlib.Path, optional Filename to store results. If not given, the filename will consist of the name of the colormap and extension '.txt'. """ - if fname is None: - fhandle = None - else: - try: - fhandle = open(fname,'w',newline='\n') - except TypeError: - fhandle = fname - labels = {'RGBA':4} if self.colors.shape[1] == 4 else {'RGB': 3} t = Table(self.colors,labels,f'Creator: {util.execution_stamp("Colormap")}') - - with open(self.name.replace(' ','_')+'.txt','w',newline='\n') if fhandle is None else fhandle as f: - t.save(f) + t.save(self._get_file_handle(fname,'txt')) def save_GOM(self,fname=None): @@ -290,26 +300,19 @@ class Colormap(mpl.colors.ListedColormap): Parameters ---------- - fname : file, str, or pathlib.Path, optional. + fname : file, str, or pathlib.Path, optional Filename to store results. If not given, the filename will consist of the name of the colormap and extension '.legend'. """ - if fname is None: - fhandle = None - else: - try: - fhandle = open(fname,'w',newline='\n') - except TypeError: - fhandle = fname # ToDo: test in GOM GOM_str = '1 1 {name} 9 {name} '.format(name=self.name.replace(" ","_")) \ + '0 1 0 3 0 0 -1 9 \\ 0 0 0 255 255 255 0 0 255 ' \ + f'30 NO_UNIT 1 1 64 64 64 255 1 0 0 0 0 0 0 3 0 {len(self.colors)}' \ + ' '.join([f' 0 {c[0]} {c[1]} {c[2]} 255 1' for c in reversed((self.colors*255).astype(int))]) \ + '\n' - with open(self.name.replace(' ','_')+'.legend','w',newline='\n') if fhandle is None else fhandle as f: - f.write(GOM_str) + + self._get_file_handle(fname,'legend').write(GOM_str) def save_gmsh(self,fname=None): @@ -318,24 +321,16 @@ class Colormap(mpl.colors.ListedColormap): Parameters ---------- - fname : file, str, or pathlib.Path, optional. + fname : file, str, or pathlib.Path, optional Filename to store results. If not given, the filename will consist of the name of the colormap and extension '.msh'. """ - if fname is None: - fhandle = None - else: - try: - fhandle = open(fname,'w',newline='\n') - except TypeError: - fhandle = fname # ToDo: test in gmsh gmsh_str = 'View.ColorTable = {\n' \ +'\n'.join([f'{c[0]},{c[1]},{c[2]},' for c in self.colors[:,:3]*255]) \ +'\n}\n' - with open(self.name.replace(' ','_')+'.msh','w',newline='\n') if fhandle is None else fhandle as f: - f.write(gmsh_str) + self._get_file_handle(fname,'msh').write(gmsh_str) @staticmethod diff --git a/python/damask/_table.py b/python/damask/_table.py index e9acb371d..68d6f94bf 100644 --- a/python/damask/_table.py +++ b/python/damask/_table.py @@ -26,7 +26,7 @@ class Table: comments_ = [comments] if isinstance(comments,str) else comments self.comments = [] if comments_ is None else [c for c in comments_] self.data = pd.DataFrame(data=data) - self.shapes = { k:(v,) if isinstance(v,(np.int,int)) else v for k,v in shapes.items() } + self.shapes = { k:(v,) if isinstance(v,(np.int64,np.int32,int)) else v for k,v in shapes.items() } self._label_uniform() def __repr__(self): From 992b4a7e6dab945b6d5dc3bb848e7ab310ffa0ab Mon Sep 17 00:00:00 2001 From: Test User Date: Thu, 11 Feb 2021 11:42:33 +0100 Subject: [PATCH 5/5] [skip ci] updated version information after successful test of v3.0.0-alpha2-421-ge96352b0e --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index b91ec55e4..a1f129287 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -v3.0.0-alpha2-415-g09c2d7f3f +v3.0.0-alpha2-421-ge96352b0e