/* Copyright 2010 Aaron J. Radke Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package cc.drx // TODO generalize to a read only version object ArrayDoubleFile{ //TODO some interface like below // def apply(size:Int, val file:File):ArrayDoubleFile // def write(ds:Iterable[Double], file:File):ArrayDoubleFile def apply(file:File,n:Int):ArrayDoubleFile = new ArrayDoubleFile(file, n) def apply(file:File):ArrayDoubleFile = apply(file, file.size.byteCount.toInt/8) } /** a minimal array like interface backed by a memory mapped file for large fast random access with minim heap memory*/ class ArrayDoubleFile(val file:File, override val size:Int) extends Iterable[Double]{ import java.io.RandomAccessFile import java.nio.channels.FileChannel import java.nio.channels.FileChannel.MapMode.{READ_WRITE, READ_ONLY} import java.nio.ByteOrder.LITTLE_ENDIAN import java.nio.DoubleBuffer private val itemSize = 8 //in bytes for 8 for Doubles private val byteSize = itemSize*size private lazy val fc = new java.io.RandomAccessFile(file.file, "rw").getChannel //TODO add a read only version mayb by a flag or inheritance // val fc = new java.io.RandomAccessFile(f.file, "r").getChannel // val byteSize = fc.size //should be 8*N = // f.size.byteCount.toInt //should be 8*N private lazy val mappedBuffer = { val mb = fc.map(READ_WRITE,0,byteSize) mb.order(LITTLE_ENDIAN) //force the use of little endian since it's an intel world mb } private lazy val buffer:DoubleBuffer = mappedBuffer.asDoubleBuffer private lazy val bufferInstance:java.nio.Buffer = buffer.asInstanceOf[java.nio.Buffer] //buffer hack from java8 private def bufferPosition(i:Int) = bufferInstance.position(i) //buffer hack from java8 //it is bad to call the position method now from java8 if compiled from jdk 9 or 10 def apply(i:Int):Double = { // asInstanceOf[Buffer] hack required to support back support to java8 when compiled from java10 bufferPosition(i) //jump into some offset location buffer.get } /**Only one of these can happen at a time*/ /* final class TakeIterator(offset:Int, n:Int) extends Iterator[Double]{ private var i:Int = 0 buffer.position(offset) //jump into some offset location def hasNext:Boolean = i < n def next():Double = {i += 1; buffer.get} } */ private val af = this /**provides the offset jump (Only one of these can happen at a time)*/ final class DropIterator(offset:Int) extends Iterator[Double]{ private var i:Int = offset bufferPosition(offset) //jump into some offset location def hasNext:Boolean = i < size def next():Double = {i += 1; buffer.get} override def size:Int = af.size - offset } final class DropIterable(offset:Int) extends Iterable[Double]{ def iterator = new DropIterator(offset) override def size:Int = af.size - offset } override def drop(offset:Int):Iterable[Double] = new DropIterable(offset) override def take(n:Int):Iterable[Double] = drop(0).take(n) def iterator:Iterator[Double] = new DropIterator(0) //TODO implement a stream get like foreach but against a slice offset jump like slice // set update equals def :=(ds:Iterable[Double]):Unit = update(0,ds) def :=(it:Iterator[Double]):Unit = update(0,it) def update(index:Int,d:Double):Unit = { bufferPosition(index) //jump into some offset location buffer.put(d) () } //**Note: you must know the size of the Iterable that it will be exhuased within the mapped write*/ def update(index:Int,ds:Iterable[Double]):Unit = { bufferPosition(index) //jump into some offset location for(d <- ds) buffer.put(d) } //**Note: you must know the size of the iterator that it will be exhuased within the mapped write*/ def update(index:Int,it:Iterator[Double]):Unit = { bufferPosition(index) //jump into some offset location while(it.hasNext) buffer.put(it.next()) } def close = fc.close }