support added for file interface. microops can be written to trace files in the ROB, and trace files can be read in runnablethread and fed into the pipelines

This commit is contained in:
Rajshekar K K 2020-09-08 23:21:54 +05:30
parent f916ed470d
commit aa5755c34e
6 changed files with 515 additions and 195 deletions

11
.gitignore vendored
View File

@ -1,9 +1,12 @@
**/bin
**/obj-pin
**/obj-comm
/.classpath
/.cproject
/.project
/.settings
**/.classpath
**/.cproject
**/.project
**/.settings
**/hs_err*
**/*.swp
**/*.swo
**/*.swn
**/*.trc

View File

@ -246,8 +246,10 @@ public class XMLParser
checkIfTraceFileAlreadyExists();
//Emulator emulator = new Emulator(EmulatorConfig.PinTool, EmulatorConfig.PinInstrumentor,
// Main.getBenchmarkArguments(), EmulatorConfig.basenameForTraceFiles);
Emulator emulator = new Emulator(EmulatorConfig.PinTool, EmulatorConfig.PinInstrumentor,
Main.getBenchmarkArguments(), EmulatorConfig.basenameForTraceFiles);
Main.getBenchmarkArguments(), "");
emulator.waitForEmulator();

View File

@ -4,22 +4,30 @@
package emulatorinterface;
import java.io.BufferedReader;
import java.io.File;
import java.io.FileInputStream;
import java.io.IOException;
import java.io.InputStreamReader;
import java.lang.reflect.Array;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.TreeMap;
import java.util.Vector;
import java.util.zip.GZIPInputStream;
import main.ArchitecturalComponent;
import main.CustomObjectPool;
import main.Main;
import memorysystem.Cache;
import memorysystem.MemorySystem;
import pipeline.PipelineInterface;
import pipeline.outoforder.OutOrderExecutionEngine;
import config.CommunicationType;
import config.EmulatorConfig;
import config.EmulatorType;
import config.MainMemoryConfig;
import config.SimulationConfig;
import config.SystemConfig;
import emulatorinterface.ThreadBlockState.blockState;
import emulatorinterface.communication.Encoding;
@ -32,9 +40,13 @@ import emulatorinterface.translator.x86.objparser.ObjParser;
import generic.BarrierTable;
import generic.CircularPacketQueue;
import generic.Core;
import generic.CustomInstructionPool;
import generic.GenericCircularQueue;
import generic.GlobalClock;
import generic.Instruction;
import generic.Operand;
import generic.OperandType;
import generic.OperationType;
import generic.Statistics;
/* MaxNumThreads threads are created from this class. Each thread
@ -56,7 +68,7 @@ public class RunnableThread implements Encoding, Runnable {
static EmulatorThreadState[] emulatorThreadState;// = new EmulatorThreadState[EMUTHREADS];
static ThreadBlockState[] threadBlockState;//=new ThreadBlockState[EMUTHREADS];
GenericCircularQueue<Instruction>[] inputToPipeline;
FileWrite obj=new FileWrite(1,EmulatorConfig.basenameForTraceFiles);
//FileWrite obj=new FileWrite(1,EmulatorConfig.basenameForTraceFiles);
// static long ignoredInstructions = 0;
// QQQ re-arrange packets for use by translate instruction.
@ -86,7 +98,8 @@ public class RunnableThread implements Encoding, Runnable {
long RAMclock;
long CoreClock;
BufferedReader[] traceReaders;
boolean[] tracesDoneReading;
/*
* This keeps on reading from the appropriate index in the shared memory
@ -95,8 +108,147 @@ public class RunnableThread implements Encoding, Runnable {
* break if the threads which started do not call threadfini in the PIN (in
* case of unclean termination). Although the problem is easily fixable.
*/
Operand retOp(String str1, String str2){
//System.out.println(str1.substring(1, str1.length()-1)+" "+str2);
Operand op= new Operand(OperandType.valueOf(str1.substring(1, str1.length()-1)),Long.parseLong(str2));
return op;
}
public void run() {
if (EmulatorConfig.emulatorType == EmulatorType.none
&& EmulatorConfig.communicationType == CommunicationType.file) {
//trace-file based simulation
for(int core = 0; core < maxCoreAssign; core++)
{
ArchitecturalComponent.getCore(core).getExecEngine().setExecutionBegun(true);
}
long microOpsReadFromFile = 0;
long microOpsSentToPipelines = 0;
while(true)
{
for(int core = 0; core < maxCoreAssign; core++)
{
if(tracesDoneReading[core] == false)
{
for(int insCount = 0; insCount < 1000; insCount++)
{
if(inputToPipeline[core].spaceLeft()==0)
break;
Instruction iNew = null;
String line = null;
try {
line = traceReaders[core].readLine();
}
catch (Exception e) {
e.printStackTrace();
misc.Error.showErrorAndExit(e.getMessage());
}
if(line==null)
iNew =Instruction.getInvalidInstruction();
else
{
microOpsReadFromFile++;
if(microOpsReadFromFile < SimulationConfig.NumInsToIgnore)
{
continue;
}
if(SimulationConfig.subsetSimulation == true
&& microOpsReadFromFile > (SimulationConfig.NumInsToIgnore + SimulationConfig.subsetSimSize))
{
iNew =Instruction.getInvalidInstruction();
line = null;
}
else
{
String[] splited = line.split(" ");
iNew = CustomObjectPool.getInstructionPool().borrowObject();
iNew.setCISCProgramCounter(Long.parseLong(splited[0]));
iNew.setOperationType(OperationType.valueOf(splited[1]));
Operand ops[] = new Operand[9];
int i=2;
for(int k=0; k<9;k++){
if(!splited[i].equals("null")){
ops[k] = retOp(splited[i], splited[i+1]);
if(k!=8) i+=2;
} else{
ops[k]=null;
if(k!=8) i+=1;
}
}
i+=1;
if(ops[0]!=null){
ops[0].setMemoryLocationFirstOperand(ops[1]);
ops[0].setMemoryLocationSecondOperand(ops[2]);
}
if(ops[3]!=null){
ops[3].setMemoryLocationFirstOperand(ops[4]);
ops[3].setMemoryLocationSecondOperand(ops[5]);
}
if(ops[6]!=null){
ops[6].setMemoryLocationFirstOperand(ops[7]);
ops[6].setMemoryLocationSecondOperand(ops[8]);
}
iNew.setSourceOperand1(ops[0]);
iNew.setSourceOperand2(ops[3]);
iNew.setDestinationOperand(ops[6]);
iNew.setSourceOperand1MemValue(Long.parseLong(splited[i]));
iNew.setSourceOperand2MemValue(Long.parseLong(splited[i+1]));
iNew.setDestinationOperandMemValue(Long.parseLong(splited[i+2]));
iNew.setBranchTargetAddress(Long.parseLong(splited[i+3]));
iNew.setBranchTaken(Boolean.parseBoolean(splited[i+4]));
//iNew.setSerialNo(Long.parseLong(splited[i+6]));
}
}
inputToPipeline[core].enqueue(iNew);
microOpsSentToPipelines++;
if(line==null)
{
tracesDoneReading[core] = true;
try
{
traceReaders[core].close();
}
catch(Exception e)
{
e.printStackTrace();
misc.Error.showErrorAndExit(e.getMessage());
}
break;
}
}
}
}
runPipelines();
boolean allTracesDoneReading = true;
for(int core = 0; core < maxCoreAssign; core++)
{
if(tracesDoneReading[core] == false)
{
allTracesDoneReading = false;
break;
}
}
if(allTracesDoneReading == true)
{
finishAllPipelines();
break;
}
}
}
else
{
//PIN-based simulation
// create pool for emulator packets
ArrayList<CircularPacketQueue> fromEmulatorAll = new ArrayList<CircularPacketQueue>(EMUTHREADS);
for(int i=0; i<EMUTHREADS; i++) {
@ -185,7 +337,7 @@ public class RunnableThread implements Encoding, Runnable {
while(fromEmulator.isEmpty() == false) {
pnew = fromEmulator.dequeue();
if(EmulatorConfig.emulatorType==EmulatorType.pin && EmulatorConfig.communicationType==CommunicationType.sharedMemory && EmulatorConfig.storeExecutionTraceInAFile==true) {
/*if(EmulatorConfig.emulatorType==EmulatorType.pin && EmulatorConfig.communicationType==CommunicationType.sharedMemory && EmulatorConfig.storeExecutionTraceInAFile==true) {
try {
try {
obj.analysisFn(javaTid, pnew.ip, pnew.value, pnew.tgt,null);
@ -201,7 +353,7 @@ public class RunnableThread implements Encoding, Runnable {
else {
}
}*/
v = pnew.value;
@ -295,6 +447,23 @@ public class RunnableThread implements Encoding, Runnable {
sorted_map.putAll(unHandledCount);
System.out.println(sorted_map);
}
if(EmulatorConfig.storeExecutionTraceInAFile == true)
{
for(int rob = 0; rob < ArchitecturalComponent.getCores().length; rob++)
{
try
{
((OutOrderExecutionEngine)ArchitecturalComponent.getCore(rob).getExecEngine()).getReorderBuffer().bw.close();
}
catch(Exception e)
{
e.printStackTrace();
misc.Error.showErrorAndExit(e.getMessage());
}
}
}
}
}
// void errorCheck(int tidApp, int emuid, int queue_size,
@ -389,6 +558,64 @@ public class RunnableThread implements Encoding, Runnable {
//added later by kush
if(SystemConfig.memControllerToUse==true)
RAMclock = (long) (1 / (SystemConfig.mainMemoryConfig.tCK) * 1000000000);
if (EmulatorConfig.emulatorType == EmulatorType.none
&& EmulatorConfig.communicationType == CommunicationType.file) {
//trace-file based simulation
int numTotalThreads = 0;
for (int benchmark=0; benchmark < Main.getTraceFileBasename().length; benchmark++)
{
for (int tid=0; ;tid++)
{
String inputFileName = Main.getTraceFileBasename()[benchmark] + "_" + tid + ".gz";
try
{
File f = new File(inputFileName);
if(f.exists() && !f.isDirectory())
{
numTotalThreads++;
}
else
{
break;
}
}
catch (Exception e)
{
break;
}
}
}
maxCoreAssign = numTotalThreads;
traceReaders = new BufferedReader[maxCoreAssign];
tracesDoneReading = new boolean[maxCoreAssign];
int index = 0;
for (int benchmark=0; benchmark < Main.getTraceFileBasename().length; benchmark++)
{
for (int tid=0; ;tid++)
{
String inputFileName = Main.getTraceFileBasename()[benchmark] + "_" + tid + ".gz";
try {
FileInputStream f = new FileInputStream(inputFileName);
GZIPInputStream g = new GZIPInputStream(f);
InputStreamReader i = new InputStreamReader(g);
traceReaders[index++] = new BufferedReader(i);
} catch (Exception e) {
if(tid==0) {
// not able to find first file is surely an error.
misc.Error.showErrorAndExit("Error in reading input packet file " + inputFileName);
} else {
System.out.println("Number of threads for benchmark " + Main.getTraceFileBasename()[benchmark] + " : " + tid);
break;
}
}
}
}
}
}
protected void runPipelines() {

View File

@ -36,6 +36,7 @@ public class FilePacket extends IpcBase implements Encoding {
String []basenameForBenchmarks;
public FilePacket(String []basenameForBenchmarks) {
/*
this.maxApplicationThreads = SystemConfig.maxNumJavaThreads*SystemConfig.numEmuThreadsPerJavaThread;
this.basenameForBenchmarks = basenameForBenchmarks;
@ -95,6 +96,7 @@ public class FilePacket extends IpcBase implements Encoding {
}
}
}
*/
}
public void initIpc() {
@ -102,7 +104,7 @@ public class FilePacket extends IpcBase implements Encoding {
}
public int fetchManyPackets(int tidApp, CircularPacketQueue fromEmulator) {
/*
if(tidApp>=maxApplicationThreads) {
misc.Error.showErrorAndExit("FilePacket cannot handle tid = " + tidApp);
}
@ -204,6 +206,8 @@ public class FilePacket extends IpcBase implements Encoding {
}
return maxSize;
*/
return 0;
}
public void errorCheck(int tidApp, long totalReads) {
@ -211,6 +215,7 @@ public class FilePacket extends IpcBase implements Encoding {
}
public void finish() {
/*
for(int i=0; i<maxApplicationThreads; i++) {
try {
if(inputBufferedReader[i] != null) {
@ -220,5 +225,6 @@ public class FilePacket extends IpcBase implements Encoding {
e.printStackTrace();
}
}
*/
}
}

View File

@ -31,6 +31,7 @@ public class Main {
public static volatile boolean statFileWritten = false;
private static String emulatorFile = " ";
private static String[] traceFileBasename;
public static int pid;
public static IpcBase ipcBase;
@ -53,23 +54,30 @@ public class Main {
Runtime.getRuntime().addShutdownHook(new ShutDownHook());
checkCommandLineArguments(arguments);
setEmulatorFile(arguments[2]);
// Read the command line arguments
String configFileName = arguments[0];
SimulationConfig.outputFileName = arguments[1];
executableAndArguments = new String[arguments.length-2];
// Parse the command line arguments
XMLParser.parse(configFileName);
printStatisticsOnAsynchronousTermination = true;
if(EmulatorConfig.communicationType==CommunicationType.file){
traceFileBasename = new String[arguments.length - 2];
for(int i = 2; i < arguments.length; i++)
setTraceFileBasename(arguments[i], i-2);
}
if(EmulatorConfig.emulatorType==EmulatorType.pin){
//pin mode
setEmulatorFile(arguments[2]);
// Read the command line arguments
executableAndArguments = new String[arguments.length-2];
// read the command line arguments for the benchmark (not emulator) here.
for(int i=2; i < arguments.length; i++) {
executableAndArguments[i-2] = arguments[i];
benchmarkArguments = benchmarkArguments + " " + arguments[i];
}
// Parse the command line arguments
XMLParser.parse(configFileName);
printStatisticsOnAsynchronousTermination = true;
}
// Initialize the statistics
Statistics.initStatistics();
@ -81,9 +89,9 @@ public class Main {
// Create a hash-table for the static representation of the executable
if(EmulatorConfig.emulatorType==EmulatorType.pin) {
ObjParser.buildStaticInstructionTable(getEmulatorFile());
} else if(EmulatorConfig.emulatorType==EmulatorType.none) {
} /*else if(EmulatorConfig.emulatorType==EmulatorType.none) {
ObjParser.initializeThreadMicroOpsList(SystemConfig.numEmuThreadsPerJavaThread);
}
}*/
ObjParser.initializeDynamicInstructionBuffer(SystemConfig.numEmuThreadsPerJavaThread*SystemConfig.numEmuThreadsPerJavaThread);
ObjParser.initializeControlMicroOps();
@ -94,12 +102,10 @@ public class Main {
// Start communication channel before starting emulator
// PS : communication channel must be started before starting the emulator
ipcBase = startCommunicationChannel();
benchmarkThreadMapping = new BenchmarkThreadMapping(getExecutableAndArguments());
benchmarkThreadMapping = new BenchmarkThreadMapping(getTraceFileBasename());
runners = new RunnableThread[SystemConfig.maxNumJavaThreads];
String emulatorArguments = constructEmulatorArguments(benchmarkArguments);
//added by harveenk
//uncomment for printing debug info
//debugPrinter = new DebugPrinter("kush_event_log");
@ -108,7 +114,10 @@ public class Main {
//outputLog = new DebugPrinter("output_log_CMDQ");
// start emulator
if(EmulatorConfig.emulatorType==EmulatorType.pin){
String emulatorArguments = constructEmulatorArguments(benchmarkArguments);
startEmulator(emulatorArguments, ipcBase);
}
for (int i=0; i<SystemConfig.maxNumJavaThreads; i++) {
runners[i] = new RunnableThread("thread"+Integer.toString(i),i, ipcBase, ArchitecturalComponent.getCores());
@ -180,7 +189,7 @@ public class Main {
ipcBase = new FilePacket(getExecutableAndArguments());
} else {
ipcBase = null;
misc.Error.showErrorAndExit("Incorrect coomunication type : " + EmulatorConfig.communicationType);
misc.Error.showErrorAndExit("Incorrect communication type : " + EmulatorConfig.communicationType);
}
return ipcBase;
@ -280,6 +289,14 @@ public class Main {
Main.emulatorFile = emulatorFile;
}
public static String[] getTraceFileBasename() {
return traceFileBasename;
}
public static void setTraceFileBasename(String inputFile, int index) {
traceFileBasename[index] = inputFile;
}
public static String[] getExecutableAndArguments() {
return executableAndArguments;
}

View File

@ -12,10 +12,16 @@ import generic.RequestType;
import generic.SimulationElement;
import generic.Statistics;
import java.io.BufferedWriter;
import java.io.File;
import java.io.FileOutputStream;
import java.io.FileWriter;
import java.io.IOException;
import java.io.OutputStreamWriter;
import java.util.zip.GZIPOutputStream;
import main.CustomObjectPool;
import config.EmulatorConfig;
import config.EnergyConfig;
import config.SimulationConfig;
@ -43,6 +49,8 @@ public class ReorderBuffer extends SimulationElement{
long numAccesses;
public BufferedWriter bw;
public ReorderBuffer(Core _core, OutOrderExecutionEngine execEngine)
{
super(PortType.Unlimited, -1, -1, -1, -1);
@ -68,6 +76,24 @@ public class ReorderBuffer extends SimulationElement{
mispredCount = 0;
branchCount = 0;
lastValidIPSeen = -1;
if(EmulatorConfig.storeExecutionTraceInAFile == true)
{
try
{
File f = new File(EmulatorConfig.basenameForTraceFiles + "_" + core.getCore_number() + ".gz");
if(f.exists() && !f.isDirectory())
{
misc.Error.showErrorAndExit("Trace file already present : " + EmulatorConfig.basenameForTraceFiles + " !!" +
"\nKindly rename the trace file and start collecting trace again.");
}
bw = new BufferedWriter(new OutputStreamWriter(new GZIPOutputStream(new FileOutputStream(EmulatorConfig.basenameForTraceFiles + "_" + core.getCore_number() + ".gz"))));
}
catch (IOException e) {
e.printStackTrace();
misc.Error.showErrorAndExit(e.getMessage());
}
}
}
//creates a new ROB entry, initialises it, and returns it
@ -257,6 +283,11 @@ public class ReorderBuffer extends SimulationElement{
execEngine.getCoreMemorySystem().issueLSQCommit(first);
}
if(EmulatorConfig.storeExecutionTraceInAFile == true)
{
writeInstructionToFile(firstInstruction);
}
//free ROB entry
retireInstructionAtHead();
@ -358,6 +389,40 @@ public class ReorderBuffer extends SimulationElement{
}
}
private void writeInstructionToFile(Instruction instruction) {
try {
Instruction tmp = instruction;
bw.write(tmp.getCISCProgramCounter()+" "+tmp.getOperationType());
if(tmp.getSourceOperand1()!=null) {
bw.write(" "+tmp.getSourceOperand1()+" "+tmp.getSourceOperand1().getMemoryLocationFirstOperand()+" "+
tmp.getSourceOperand1().getMemoryLocationSecondOperand());
}
else bw.write(" null null null");
if(tmp.getSourceOperand2()!=null) {
bw.write(" "+tmp.getSourceOperand2()+" "+tmp.getSourceOperand2().getMemoryLocationFirstOperand()+" "+
tmp.getSourceOperand2().getMemoryLocationSecondOperand());
}
else bw.write(" null null null");
if(tmp.getDestinationOperand()!=null) {
bw.write(" "+tmp.getDestinationOperand()+" "+tmp.getDestinationOperand().getMemoryLocationFirstOperand()+" "+
tmp.getDestinationOperand().getMemoryLocationSecondOperand());
}
else bw.write(" null null null");
bw.write(" "+tmp.getSourceOperand1MemValue()+" "+
tmp.getSourceOperand2MemValue()+" "+tmp.getDestinationOperandMemValue()+" "+
tmp.getBranchTargetAddress()+" "+tmp.isBranchTaken()+" "+tmp.getThreadID()+" "+tmp.getSerialNo()+"\n");
}
catch(Exception e)
{
e.printStackTrace();
misc.Error.showErrorAndExit(e.getMessage());
}
}
//debug helper - print contents of ROB
public void dump()
{