From d1ba1dd5f7d99626b903c279bcf364e6d1259d11 Mon Sep 17 00:00:00 2001 From: stepan <stepan.sindelar@oracle.com> Date: Mon, 6 Nov 2017 18:19:43 +0100 Subject: [PATCH] Simplify object size calculation --- .../truffle/r/library/utils/ObjectSize.java | 12 +- .../oracle/truffle/r/library/utils/Rprof.java | 4 +- .../truffle/r/library/utils/Rprofmem.java | 32 +-- .../com/oracle/truffle/r/runtime/RError.java | 3 +- .../runtime/data/AgentObjectSizeFactory.java | 215 --------------- .../r/runtime/data/ObjectSizeFactory.java | 85 ------ .../data/OutputAgentObjectSizeFactory.java | 56 ---- .../truffle/r/runtime/data/RDataFactory.java | 23 +- .../truffle/r/runtime/data/RObjectSize.java | 254 ++++++++++++------ .../runtime/data/SimpleObjectSizeFactory.java | 74 ----- .../truffle/r/test/ExpectedTestOutput.test | 12 + .../test/builtins/TestBuiltin_objectsize.java | 34 +-- documentation/dev/managed_ffi.md | 9 +- 13 files changed, 209 insertions(+), 604 deletions(-) delete mode 100644 com.oracle.truffle.r.runtime/src/com/oracle/truffle/r/runtime/data/AgentObjectSizeFactory.java delete mode 100644 com.oracle.truffle.r.runtime/src/com/oracle/truffle/r/runtime/data/ObjectSizeFactory.java delete mode 100644 com.oracle.truffle.r.runtime/src/com/oracle/truffle/r/runtime/data/OutputAgentObjectSizeFactory.java delete mode 100644 com.oracle.truffle.r.runtime/src/com/oracle/truffle/r/runtime/data/SimpleObjectSizeFactory.java rename com.oracle.truffle.r.runtime/src/com/oracle/truffle/r/runtime/data/ObjSizeAgent.java => com.oracle.truffle.r.test/src/com/oracle/truffle/r/test/builtins/TestBuiltin_objectsize.java (53%) diff --git a/com.oracle.truffle.r.library/src/com/oracle/truffle/r/library/utils/ObjectSize.java b/com.oracle.truffle.r.library/src/com/oracle/truffle/r/library/utils/ObjectSize.java index 6e32b3837a..1385ee014b 100644 --- a/com.oracle.truffle.r.library/src/com/oracle/truffle/r/library/utils/ObjectSize.java +++ b/com.oracle.truffle.r.library/src/com/oracle/truffle/r/library/utils/ObjectSize.java @@ -28,7 +28,6 @@ import com.oracle.truffle.api.CompilerDirectives.TruffleBoundary; import com.oracle.truffle.api.dsl.Fallback; import com.oracle.truffle.api.dsl.Specialization; import com.oracle.truffle.r.nodes.builtin.RExternalBuiltinNode; -import com.oracle.truffle.r.runtime.data.RNull; import com.oracle.truffle.r.runtime.data.RObjectSize; import com.oracle.truffle.r.runtime.data.RTypedValue; @@ -43,15 +42,6 @@ public abstract class ObjectSize extends RExternalBuiltinNode.Arg1 { noCasts(ObjectSize.class); } - private static class MyIgnoreObjectHandler implements RObjectSize.IgnoreObjectHandler { - @Override - public boolean ignore(Object rootObject, Object obj) { - return obj == RNull.instance; - } - } - - private static final MyIgnoreObjectHandler ignoreObjectHandler = new MyIgnoreObjectHandler(); - @Specialization protected int objectSize(@SuppressWarnings("unused") int o) { return RObjectSize.INT_SIZE; @@ -70,6 +60,6 @@ public abstract class ObjectSize extends RExternalBuiltinNode.Arg1 { @Fallback @TruffleBoundary protected int objectSize(Object o) { - return (int) RObjectSize.getObjectSize(o, ignoreObjectHandler); + return (int) (RObjectSize.getRecursiveObjectSize(o) / 8L); } } diff --git a/com.oracle.truffle.r.library/src/com/oracle/truffle/r/library/utils/Rprof.java b/com.oracle.truffle.r.library/src/com/oracle/truffle/r/library/utils/Rprof.java index 6d8e31c50f..e343e52ff6 100644 --- a/com.oracle.truffle.r.library/src/com/oracle/truffle/r/library/utils/Rprof.java +++ b/com.oracle.truffle.r.library/src/com/oracle/truffle/r/library/utils/Rprof.java @@ -140,7 +140,7 @@ public abstract class Rprof extends RExternalBuiltinNode.Arg8 implements MemoryC if (profState.memoryQuad == null) { return; } - long size = RObjectSize.getObjectSize(data, Rprofmem.myIgnoreObjectHandler); + long size = RObjectSize.getObjectSize(data); if (data instanceof RAbstractVector) { if (size >= Rprofmem.LARGE_VECTOR) { profState.memoryQuad.largeV += size; @@ -157,7 +157,7 @@ public abstract class Rprof extends RExternalBuiltinNode.Arg8 implements MemoryC @TruffleBoundary public void reportCopying(RAbstractVector source, RAbstractVector dest) { RprofState profState = RprofState.get(); - profState.memoryQuad.copied += RObjectSize.getObjectSize(source, Rprofmem.myIgnoreObjectHandler); + profState.memoryQuad.copied += RObjectSize.getObjectSize(source); } private static void endProfiling() { diff --git a/com.oracle.truffle.r.library/src/com/oracle/truffle/r/library/utils/Rprofmem.java b/com.oracle.truffle.r.library/src/com/oracle/truffle/r/library/utils/Rprofmem.java index 197dc7b9bd..447f1baeda 100644 --- a/com.oracle.truffle.r.library/src/com/oracle/truffle/r/library/utils/Rprofmem.java +++ b/com.oracle.truffle.r.library/src/com/oracle/truffle/r/library/utils/Rprofmem.java @@ -22,6 +22,10 @@ */ package com.oracle.truffle.r.library.utils; +import static com.oracle.truffle.r.nodes.builtin.CastBuilder.Predef.doubleValue; +import static com.oracle.truffle.r.nodes.builtin.CastBuilder.Predef.instanceOf; +import static com.oracle.truffle.r.nodes.builtin.CastBuilder.Predef.stringValue; + import java.io.FileOutputStream; import java.io.IOException; import java.io.PrintStream; @@ -29,10 +33,6 @@ import java.io.PrintStream; import com.oracle.truffle.api.CompilerDirectives.TruffleBoundary; import com.oracle.truffle.api.dsl.Specialization; import com.oracle.truffle.api.frame.Frame; -import com.oracle.truffle.api.nodes.Node; -import static com.oracle.truffle.r.nodes.builtin.CastBuilder.Predef.doubleValue; -import static com.oracle.truffle.r.nodes.builtin.CastBuilder.Predef.instanceOf; -import static com.oracle.truffle.r.nodes.builtin.CastBuilder.Predef.stringValue; import com.oracle.truffle.r.nodes.builtin.RExternalBuiltinNode; import com.oracle.truffle.r.runtime.RArguments; import com.oracle.truffle.r.runtime.RError; @@ -93,28 +93,6 @@ public abstract class Rprofmem extends RExternalBuiltinNode.Arg3 { private static final int PAGE_SIZE = 2000; static final int LARGE_VECTOR = 128; - /** - * We ignore nested {@link RTypedValue} instances as these will have been counted already. We - * also ignore {@link Node} instances, except in {@link RFunction} objects. - */ - private static class MyIgnoreObjectHandler implements RObjectSize.IgnoreObjectHandler { - @Override - public boolean ignore(Object rootObject, Object obj) { - if (obj == RNull.instance) { - return true; - } else { - Class<?> klass = obj.getClass(); - if (RTypedValue.class.isAssignableFrom(klass)) { - return true; - } else { - return false; - } - } - } - } - - static final RObjectSize.IgnoreObjectHandler myIgnoreObjectHandler = new MyIgnoreObjectHandler(); - private static final RDataFactory.Listener LISTENER = new RDataFactory.Listener() { @Override public void reportAllocation(RTypedValue data) { @@ -132,7 +110,7 @@ public abstract class Rprofmem extends RExternalBuiltinNode.Arg3 { } String name = func.getRootNode().getName(); - long size = RObjectSize.getObjectSize(data, myIgnoreObjectHandler); + long size = RObjectSize.getObjectSize(data); if (data instanceof RAbstractVector && size >= LARGE_VECTOR) { if (size > profmemState.threshold) { profmemState.out().printf("%d: %s\n", size, name); diff --git a/com.oracle.truffle.r.runtime/src/com/oracle/truffle/r/runtime/RError.java b/com.oracle.truffle.r.runtime/src/com/oracle/truffle/r/runtime/RError.java index d8cde8b181..bae562ba67 100644 --- a/com.oracle.truffle.r.runtime/src/com/oracle/truffle/r/runtime/RError.java +++ b/com.oracle.truffle.r.runtime/src/com/oracle/truffle/r/runtime/RError.java @@ -906,7 +906,8 @@ public final class RError extends RuntimeException implements TruffleException { OS_REQUEST_LOCALE("OS reports request to set locale to \"%s\" cannot be honored"), INVALID_TYPE("invalid type (%s) for '%s' (must be a %s)"), NOT_A_LIST_OF_SOCKETS("not a list of sockets"), - NOT_A_SOCKET_CONNECTION("not a socket connection"); + NOT_A_SOCKET_CONNECTION("not a socket connection"), + UNEXPECTED_OBJ_IN_SIZE("Unexpected object type %s while calculating estimated object size."); public final String message; final boolean hasArgs; diff --git a/com.oracle.truffle.r.runtime/src/com/oracle/truffle/r/runtime/data/AgentObjectSizeFactory.java b/com.oracle.truffle.r.runtime/src/com/oracle/truffle/r/runtime/data/AgentObjectSizeFactory.java deleted file mode 100644 index bab299694f..0000000000 --- a/com.oracle.truffle.r.runtime/src/com/oracle/truffle/r/runtime/data/AgentObjectSizeFactory.java +++ /dev/null @@ -1,215 +0,0 @@ -/* - * Copyright (c) 2016, 2017, Oracle and/or its affiliates. All rights reserved. - * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. - * - * This code is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License version 2 only, as - * published by the Free Software Foundation. - * - * This code is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - * version 2 for more details (a copy is included in the LICENSE file that - * accompanied this code). - * - * You should have received a copy of the GNU General Public License version - * 2 along with this work; if not, write to the Free Software Foundation, - * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. - * - * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA - * or visit www.oracle.com if you need additional information or have any - * questions. - */ -package com.oracle.truffle.r.runtime.data; - -import java.io.FileOutputStream; -import java.io.IOException; -import java.io.InputStream; -import java.lang.management.ManagementFactory; -import java.lang.reflect.Array; -import java.lang.reflect.Field; -import java.lang.reflect.Method; -import java.lang.reflect.Modifier; -import java.nio.file.Files; -import java.nio.file.Path; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.HashSet; -import java.util.Map; -import java.util.jar.Attributes; -import java.util.jar.JarEntry; -import java.util.jar.JarOutputStream; -import java.util.jar.Manifest; - -import javax.tools.ToolProvider; - -import com.oracle.truffle.r.runtime.RInternalError; -import com.oracle.truffle.r.runtime.RRuntime; -import com.oracle.truffle.r.runtime.Utils; -import com.oracle.truffle.r.runtime.data.RObjectSize.IgnoreObjectHandler; -import com.oracle.truffle.r.runtime.data.RObjectSize.TypeCustomizer; - -/** - * Uses an instrumentation agent to get an accurate estimate of an objects size, plus reflection to - * aggregate the size of object-valued fields. Sharing is not handled in the general case, although - * some special cases are handed, such as the fact that {@link RNull} is a singleton. - * - * In order to satisfy the requirements of the Java instrumentation API, we have to load the agent - * from a jar file. The creation and loading is all orchestrated by this class. - */ -public class AgentObjectSizeFactory extends ObjectSizeFactory { - - private final Map<Class<?>, ArrayList<Field>> objectFieldsMap = new HashMap<>(); - - public AgentObjectSizeFactory() { - if (!ObjSizeAgent.isInitialized()) { - try { - createAgentJar(); - } catch (Exception ex) { - // not available - Utils.rSuicide("failed to load ObjSizeAgent: " + ex.getMessage()); - } - } - } - - /** - * Adds the class file bytes for a given class to a JAR stream. - */ - static void add(JarOutputStream jar, Class<?> c) throws IOException { - String name = c.getName(); - String classAsPath = name.replace('.', '/') + ".class"; - jar.putNextEntry(new JarEntry(classAsPath)); - - InputStream stream = c.getClassLoader().getResourceAsStream(classAsPath); - - int nRead; - byte[] buf = new byte[1024]; - while ((nRead = stream.read(buf, 0, buf.length)) != -1) { - jar.write(buf, 0, nRead); - } - - jar.closeEntry(); - } - - protected void createAgentJar() throws Exception { - Manifest manifest = new Manifest(); - manifest.getMainAttributes().put(Attributes.Name.MANIFEST_VERSION, "1.0"); - Attributes mainAttrs = manifest.getMainAttributes(); - mainAttrs.putValue("Agent-Class", ObjSizeAgent.class.getName()); - mainAttrs.putValue("Premain-Class", ObjSizeAgent.class.getName()); - - Path jar = Files.createTempFile("myagent", ".jar"); - try { - JarOutputStream jarStream = new JarOutputStream(new FileOutputStream(jar.toFile()), manifest); - add(jarStream, ObjSizeAgent.class); - jarStream.close(); - - loadAgent(jar); - } finally { - Files.deleteIfExists(jar); - } - } - - public static void loadAgent(Path agent) throws Exception { - String vmName = ManagementFactory.getRuntimeMXBean().getName(); - int p = vmName.indexOf('@'); - String pid = vmName.substring(0, p); - ClassLoader cl = ToolProvider.getSystemToolClassLoader(); - Class<?> c = Class.forName("com.sun.tools.attach.VirtualMachine", true, cl); - Method attach = c.getDeclaredMethod("attach", String.class); - Method loadAgent = c.getDeclaredMethod("loadAgent", String.class, String.class); - Object vm = attach.invoke(null, pid); - loadAgent.invoke(vm, agent.toString(), ""); - } - - @Override - public long getObjectSize(Object obj, IgnoreObjectHandler ignoreObjectHandler) { - return getObjectSize(obj, obj, ignoreObjectHandler, new HashSet<>()); - } - - private long getObjectSize(Object rootObj, Object obj, IgnoreObjectHandler ignoreObjectHandler, HashSet<Object> visited) { - if (visited.contains(obj)) { - return 0; - } - visited.add(obj); - try { - long basicSize = ObjSizeAgent.objectSize(obj); - long size = basicSize; - Class<?> klass = obj.getClass(); - if (klass.isArray() && !klass.getComponentType().isPrimitive()) { - for (int i = 0; i < Array.getLength(obj); i++) { - Object elem = Array.get(obj, i); - if (elem == null || isNa(elem)) { - continue; - } else { - size += getObjectSize(rootObj, elem, ignoreObjectHandler, visited); - } - } - } else { - ArrayList<Field> objectFields = objectFieldsMap.get(klass); - if (objectFields == null) { - objectFields = new ArrayList<>(); - findObjectFields(obj.getClass(), objectFields); - objectFieldsMap.put(klass, objectFields); - } - for (Field objectField : objectFields) { - Object fieldObj = objectField.get(obj); - if (fieldObj == null || ignoreObjectHandler.ignore(rootObj, fieldObj)) { - continue; - } else { - TypeCustomizer typeCustomizer = getCustomizer(fieldObj.getClass()); - if (typeCustomizer == null) { - size += getObjectSize(rootObj, fieldObj, ignoreObjectHandler, visited); - } else { - size += typeCustomizer.getObjectSize(fieldObj); - } - } - } - } - return size; - } catch (Throwable t) { - throw RInternalError.shouldNotReachHere(t); - } - } - - private static boolean isNa(Object elem) { - String typeName = elem.getClass().getSimpleName(); - switch (typeName) { - case "Integer": - return RRuntime.isNA((int) elem); - case "Double": - return RRuntime.isNA((double) elem); - case "String": - return RRuntime.isNA((String) elem); - default: - return false; - } - } - - /** - * Walks the superclass hierarchy of {@code klass} and accumulates all object-valued fields in - * {@code objectFields}. - */ - private static void findObjectFields(Class<?> klass, ArrayList<Field> objectFields) { - if (klass != Object.class) { - findObjectFields(klass.getSuperclass(), objectFields); - Field[] fields = klass.getDeclaredFields(); - for (Field field : fields) { - Class<?> fieldClass = field.getType(); - if (fieldClass.isPrimitive()) { - continue; - } - int modifiers = field.getModifiers(); - if (Modifier.isStatic(modifiers)) { - continue; - } - // check for special case of an completely ignored type - if (getCustomizer(fieldClass) == RObjectSize.IGNORE) { - continue; - } - field.setAccessible(true); - objectFields.add(field); - } - } - } -} diff --git a/com.oracle.truffle.r.runtime/src/com/oracle/truffle/r/runtime/data/ObjectSizeFactory.java b/com.oracle.truffle.r.runtime/src/com/oracle/truffle/r/runtime/data/ObjectSizeFactory.java deleted file mode 100644 index 9de56a9bce..0000000000 --- a/com.oracle.truffle.r.runtime/src/com/oracle/truffle/r/runtime/data/ObjectSizeFactory.java +++ /dev/null @@ -1,85 +0,0 @@ -/* - * Copyright (c) 2016, 2017, Oracle and/or its affiliates. All rights reserved. - * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. - * - * This code is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License version 2 only, as - * published by the Free Software Foundation. - * - * This code is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - * version 2 for more details (a copy is included in the LICENSE file that - * accompanied this code). - * - * You should have received a copy of the GNU General Public License version - * 2 along with this work; if not, write to the Free Software Foundation, - * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. - * - * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA - * or visit www.oracle.com if you need additional information or have any - * questions. - */ -package com.oracle.truffle.r.runtime.data; - -import java.util.ArrayList; - -import com.oracle.truffle.r.runtime.FastRConfig; -import com.oracle.truffle.r.runtime.data.RObjectSize.IgnoreObjectHandler; -import com.oracle.truffle.r.runtime.data.RObjectSize.TypeCustomizer; - -public abstract class ObjectSizeFactory { - - private static ArrayList<TypeCustomizerData> typeCustomizers = new ArrayList<>(); // system wide - - static { - String prop = System.getProperty("fastr.objectsize.factory.class"); - if (prop == null) { - if (FastRConfig.ManagedMode) { - prop = SimpleObjectSizeFactory.class.getName(); - } else { - prop = AgentObjectSizeFactory.class.getName(); - } - } - try { - theInstance = (ObjectSizeFactory) Class.forName(prop).newInstance(); - } catch (Exception ex) { - // CheckStyle: stop system..print check - System.err.println("Failed to instantiate class: " + prop); - } - } - - private static ObjectSizeFactory theInstance; - - public static ObjectSizeFactory getInstance() { - return theInstance; - } - - /** - * See {@link RObjectSize#getObjectSize}. - */ - public abstract long getObjectSize(Object obj, IgnoreObjectHandler ignoreObjectHandler); - - public void registerTypeCustomizer(Class<?> klass, TypeCustomizer typeCustomizer) { - typeCustomizers.add(new TypeCustomizerData(klass, typeCustomizer)); - } - - protected static TypeCustomizer getCustomizer(Class<?> objClass) { - for (TypeCustomizerData customizer : typeCustomizers) { - if (customizer.type.isAssignableFrom(objClass)) { - return customizer.customizer; - } - } - return null; - } - - private static final class TypeCustomizerData { - private final TypeCustomizer customizer; - private final Class<?> type; - - private TypeCustomizerData(Class<?> type, TypeCustomizer customizer) { - this.customizer = customizer; - this.type = type; - } - } -} diff --git a/com.oracle.truffle.r.runtime/src/com/oracle/truffle/r/runtime/data/OutputAgentObjectSizeFactory.java b/com.oracle.truffle.r.runtime/src/com/oracle/truffle/r/runtime/data/OutputAgentObjectSizeFactory.java deleted file mode 100644 index 9fbc0f2301..0000000000 --- a/com.oracle.truffle.r.runtime/src/com/oracle/truffle/r/runtime/data/OutputAgentObjectSizeFactory.java +++ /dev/null @@ -1,56 +0,0 @@ -/* - * Copyright (c) 2016, 2017, Oracle and/or its affiliates. All rights reserved. - * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. - * - * This code is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License version 2 only, as - * published by the Free Software Foundation. - * - * This code is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - * version 2 for more details (a copy is included in the LICENSE file that - * accompanied this code). - * - * You should have received a copy of the GNU General Public License version - * 2 along with this work; if not, write to the Free Software Foundation, - * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. - * - * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA - * or visit www.oracle.com if you need additional information or have any - * questions. - */ -package com.oracle.truffle.r.runtime.data; - -import java.io.FileWriter; -import java.io.IOException; -import java.io.PrintWriter; - -import com.oracle.truffle.r.runtime.Utils; -import com.oracle.truffle.r.runtime.data.RObjectSize.IgnoreObjectHandler; - -/** - * A debugging tool, logs all calls to {@link #getObjectSize(Object, IgnoreObjectHandler)} to a - * file. - * - */ -public class OutputAgentObjectSizeFactory extends AgentObjectSizeFactory { - - private PrintWriter printWriter; - - public OutputAgentObjectSizeFactory() { - try { - printWriter = new PrintWriter(new FileWriter(Utils.getLogPath("fastr_objectsize.log").toString())); - } catch (IOException ex) { - Utils.rSuicide(ex.getMessage()); - } - } - - @Override - public long getObjectSize(Object obj, IgnoreObjectHandler ignoreObjectHandler) { - long size = super.getObjectSize(obj, ignoreObjectHandler); - printWriter.printf("%s: %d\n", obj.getClass().getSimpleName(), (int) size); - printWriter.flush(); - return size; - } -} diff --git a/com.oracle.truffle.r.runtime/src/com/oracle/truffle/r/runtime/data/RDataFactory.java b/com.oracle.truffle.r.runtime/src/com/oracle/truffle/r/runtime/data/RDataFactory.java index 47671dfdec..be21c056a3 100644 --- a/com.oracle.truffle.r.runtime/src/com/oracle/truffle/r/runtime/data/RDataFactory.java +++ b/com.oracle.truffle.r.runtime/src/com/oracle/truffle/r/runtime/data/RDataFactory.java @@ -44,7 +44,6 @@ import com.oracle.truffle.r.runtime.builtins.RBuiltinDescriptor; import com.oracle.truffle.r.runtime.context.RContext; import com.oracle.truffle.r.runtime.data.RPromise.EagerFeedback; import com.oracle.truffle.r.runtime.data.RPromise.PromiseState; -import com.oracle.truffle.r.runtime.data.model.RAbstractVector; import com.oracle.truffle.r.runtime.env.REnvironment; import com.oracle.truffle.r.runtime.ffi.DLL.SymbolHandle; import com.oracle.truffle.r.runtime.gnur.SEXPTYPE; @@ -706,26 +705,6 @@ public final class RDataFactory { } private static long getSize(RTypedValue data) { - long multiplier = 8; - switch (data.getRType()) { - case Complex: - multiplier = 16; - break; - case Integer: - multiplier = 4; - break; - case Logical: - case Raw: - multiplier = 1; - break; - } - if (data instanceof RSequence) { - return 32 + 2 * multiplier; - } else if (data instanceof RAbstractVector) { - return 32 + ((RAbstractVector) data).getLength() * multiplier; - } else { - // take a default value for non-vector objects - return 64; - } + return RObjectSize.getObjectSize(data); } } diff --git a/com.oracle.truffle.r.runtime/src/com/oracle/truffle/r/runtime/data/RObjectSize.java b/com.oracle.truffle.r.runtime/src/com/oracle/truffle/r/runtime/data/RObjectSize.java index 230fc8a0be..c4a0c7f23c 100644 --- a/com.oracle.truffle.r.runtime/src/com/oracle/truffle/r/runtime/data/RObjectSize.java +++ b/com.oracle.truffle.r.runtime/src/com/oracle/truffle/r/runtime/data/RObjectSize.java @@ -22,35 +22,30 @@ */ package com.oracle.truffle.r.runtime.data; -import com.oracle.truffle.api.Assumption; -import com.oracle.truffle.api.CallTarget; -import com.oracle.truffle.api.frame.Frame; -import com.oracle.truffle.api.frame.FrameDescriptor; -import com.oracle.truffle.api.nodes.Node; -import com.oracle.truffle.r.runtime.RCaller; -import com.oracle.truffle.r.runtime.builtins.RBuiltinDescriptor; -import com.oracle.truffle.r.runtime.gnur.SEXPTYPE; +import java.util.ArrayDeque; +import java.util.HashSet; + +import com.oracle.truffle.api.CompilerDirectives.TruffleBoundary; +import com.oracle.truffle.api.interop.TruffleObject; +import com.oracle.truffle.api.object.DynamicObject; +import com.oracle.truffle.api.object.Property; +import com.oracle.truffle.api.object.Shape; +import com.oracle.truffle.r.runtime.ArgumentsSignature; +import com.oracle.truffle.r.runtime.RError; +import com.oracle.truffle.r.runtime.RError.Message; +import com.oracle.truffle.r.runtime.data.model.RAbstractComplexVector; +import com.oracle.truffle.r.runtime.data.model.RAbstractDoubleVector; +import com.oracle.truffle.r.runtime.data.model.RAbstractIntVector; +import com.oracle.truffle.r.runtime.data.model.RAbstractListVector; +import com.oracle.truffle.r.runtime.data.model.RAbstractLogicalVector; +import com.oracle.truffle.r.runtime.data.model.RAbstractRawVector; +import com.oracle.truffle.r.runtime.data.model.RAbstractStringVector; +import com.oracle.truffle.r.runtime.data.model.RAbstractVector; +import com.oracle.truffle.r.runtime.env.REnvironment; /** * Support for the sizing of the objects that flow through the interpreter, i.e., mostly - * {@link RTypedValue}, but also including scalar types like {@code String} and dimension data for - * arrays, i.e., {@code int[]}. - * - * The actually implementation is controlled by {@link ObjectSizeFactory} to finesse problems with - * Java VMs that do not support reflection. - * - * Owing to the (implementation) complexity of some of the types, two levels of customization are - * provided: - * <ol> - * <li>A completely custom sizing implementation can be provided for a specific type. This effects - * all sizing computations.</li> - * <li>In any given call to {@link #getObjectSize} an instance of {@IgnoreObjectHandler} can passed. - * This allows some additional dynamic control over certain fields depending of the context of the - * call. For example, when tracking the incremental memory allocation via {@link RDataFactory}, we - * do not want to (double) count fields of type {@link RTypedValue}. However, when computing the - * total size of the object, e.g. for the {@code utils::object.size} builtin, we do want to count - * them.</li> - * </ol> + * {@link RTypedValue}, but also including scalar types like {@code String}. * */ public class RObjectSize { @@ -58,74 +53,173 @@ public class RObjectSize { public static final int DOUBLE_SIZE = 64; public static final int BYTE_SIZE = 8; - public interface TypeCustomizer { - /** - * Allows complete control over sizing of a type registered with - * {@link #registerTypeCustomizer}. - */ - long getObjectSize(Object obj); - } - - public interface IgnoreObjectHandler { - /** - * Controls which fields of an object passed to {@link #getObjectSize} will be ignored. - * {@code rootObject} is the initiating object and {@code obj} is some field of that object - * or one of its components. The return value should be {@code true} if {@code obj} should - * be ignored. - */ - boolean ignore(Object rootObject, Object obj); - } + private static final int CHAR_SIZE = 16; + private static final int OBJECT_SIZE = 64; /** - * Returns an estimate of the size of the this object, including the size of any object-valued - * fields, recursively. Evidently this is a snapshot and the size can change as, e.g., + * Returns an estimate of the size of the this object in bits, excluding the size of any + * object-valued fields. Evidently this is a snapshot and the size can change as, e.g., * attributes are added/removed. * - * If called immediately after creation by {@link RDataFactory}, with an - * {@link IgnoreObjectHandler} that ignores objects created separately and, it provides an - * approximation of the incremental memory usage of the system. - * - * @param ignoreObjectHandler An object that is called to decide whether to include the - * contribution a field of this object (and its sub-objects) in the result. Passing - * {@code null} includes everything. N.B. {@code obj} is typed as {@code Object} only - * to allow scalar typed such as {@code String} to be passed. - * + * If called immediately after creation by {@link RDataFactory} provides an approximation of the + * incremental memory usage of the system. */ - public static long getObjectSize(Object obj, IgnoreObjectHandler ignoreObjectHandler) { - return (int) ObjectSizeFactory.getInstance().getObjectSize(obj, ignoreObjectHandler); + @TruffleBoundary + public static long getObjectSize(Object obj) { + return getObjectSizeImpl(obj); } /** - * Register a {@link TypeCustomizer} for {@code klass} and its subclasses. I.e. and object - * {@code obj} is customized iff {@code klass.isAssignableFrom(obj.getClass())}. + * Returns an estimate of the size of the this object in bits, including the size of any + * object-valued fields, recursively. Evidently this is a snapshot and the size can change as, + * e.g., attributes are added/removed. */ - public static void registerTypeCustomizer(Class<?> klass, TypeCustomizer typeCustomizer) { - ObjectSizeFactory.getInstance().registerTypeCustomizer(klass, typeCustomizer); + @TruffleBoundary + public static long getRecursiveObjectSize(Object target) { + ArrayDeque<Object> stack = new ArrayDeque<>(); + HashSet<Object> visited = new HashSet<>(); + stack.push(target); + visited.add(target); + + long result = 0; + while (!stack.isEmpty()) { + Object obj = stack.pop(); + result += getObjectSizeImpl(obj); + if (obj != null) { + pushReferences(stack, visited, obj); + } + } + return result; } - /** - * This denotes a special customizer that completely ignores instances of the type and its - * subclasses. It allows a more efficient implementation as the type can be suppressed - * completely from the computation at the time fields of a containing type are analyzed. - */ - public static final TypeCustomizer IGNORE = new TypeCustomizer() { + private static void pushReferences(ArrayDeque<Object> stack, HashSet<Object> visited, Object obj) { + if (obj instanceof RAttributable) { + DynamicObject attrs = ((RAttributable) obj).getAttributes(); + if (attrs != null) { + Shape shape = attrs.getShape(); + for (Property prop : shape.getProperties()) { + Object propVal = prop.get(attrs, shape); + pushIfNotPresent(stack, visited, propVal); + } + } + } + if (obj instanceof RAbstractListVector) { + RAbstractListVector list = (RAbstractListVector) obj; + for (int i = 0; i < list.getLength(); i++) { + pushIfNotPresent(stack, visited, list.getDataAt(i)); + } + } else if (obj instanceof RArgsValuesAndNames) { + RArgsValuesAndNames args = (RArgsValuesAndNames) obj; + for (int i = 0; i < args.getLength(); i++) { + pushIfNotPresent(stack, visited, args.getArgument(i)); + } + } + // Note: environments are ignored + } - @Override - public long getObjectSize(Object obj) { + private static void pushIfNotPresent(ArrayDeque<Object> stack, HashSet<Object> visited, Object obj) { + if (!visited.contains(obj)) { + stack.push(obj); + visited.add(obj); + } + } + + private static long getObjectSizeImpl(Object obj) { + // Note: if this gets too complex, it may be replaced by a system of providers or getSize + // abstract method on RTypedValue. For now, we do not want to add yet another abstract + // method to already complicated hierarchy and providers would only mean OO version of the + // same code below. + if (obj == null) { + return 0; + } + // Primitive types: + if (obj instanceof Integer) { + return INT_SIZE; + } else if (obj instanceof Double) { + return DOUBLE_SIZE; + } else if (obj instanceof Byte) { + return BYTE_SIZE; + } else if (obj instanceof String) { + return CHAR_SIZE * ((String) obj).length(); + } + // Check that we have RTypedValue: + if (!(obj instanceof RTypedValue)) { + // We ignore objects from other languages for now + if (!(obj instanceof TruffleObject)) { + reportWarning(obj); + } + return 0; + } + long attributesSize = 0; + if (obj instanceof RAttributable) { + DynamicObject attrs = ((RAttributable) obj).getAttributes(); + if (attrs != null) { + attributesSize = attrs.size() * OBJECT_SIZE; + } + } + // Individual RTypedValues: + if (obj instanceof RPromise || obj instanceof RAbstractListVector || obj instanceof REnvironment || obj instanceof RExternalPtr || obj instanceof RFunction) { + // promise: there is no value allocated yet, we may use the size of the closure + return OBJECT_SIZE + attributesSize; + } else if (obj instanceof RStringSequence) { + RStringSequence seq = (RStringSequence) obj; + if (seq.getLength() == 0) { + return OBJECT_SIZE + INT_SIZE * 2; // we cannot get prefix/suffix... + } else { + return OBJECT_SIZE + seq.getDataAt(0).length() * CHAR_SIZE; + } + } else if (obj instanceof RSequence) { + // count: start, stride, length + return OBJECT_SIZE + 2 * getElementSize((RAbstractVector) obj) + INT_SIZE + attributesSize; + } else if (obj instanceof RAbstractStringVector) { + RAbstractStringVector strVec = (RAbstractStringVector) obj; + long result = OBJECT_SIZE; + for (int i = 0; i < strVec.getLength(); i++) { + result += strVec.getDataAt(i).length() * CHAR_SIZE; + } + return result + attributesSize; + } else if (obj instanceof RAbstractVector) { + RAbstractVector vec = (RAbstractVector) obj; + return OBJECT_SIZE + getElementSize(vec) * vec.getLength() + attributesSize; + } else if (obj instanceof RScalar) { + // E.g. singletons RNull or REmpty. RInteger, RLogical etc. already caught by + // RAbstractVector branch return 0; + } else if (obj instanceof RArgsValuesAndNames) { + return getArgsAndValuesSize((RArgsValuesAndNames) obj); + } else { + reportWarning(obj); + return OBJECT_SIZE; } - }; + } + + private static int getElementSize(RAbstractVector vector) { + if (vector instanceof RAbstractDoubleVector) { + return DOUBLE_SIZE; + } else if (vector instanceof RAbstractIntVector) { + return INT_SIZE; + } else if (vector instanceof RAbstractLogicalVector || vector instanceof RAbstractRawVector) { + return BYTE_SIZE; + } else if (vector instanceof RAbstractComplexVector) { + return DOUBLE_SIZE * 2; + } + reportWarning(vector); + return INT_SIZE; + } + + private static long getArgsAndValuesSize(RArgsValuesAndNames args) { + long result = OBJECT_SIZE + args.getLength() * OBJECT_SIZE; + ArgumentsSignature signature = args.getSignature(); + for (int i = 0; i < signature.getLength(); i++) { + String name = signature.getName(i); + if (name != null) { + result += name.length() * CHAR_SIZE; + } + } + return result; + } - // TODO construct proper customizers for some of these. - static { - registerTypeCustomizer(Frame.class, IGNORE); - registerTypeCustomizer(FrameDescriptor.class, IGNORE); - registerTypeCustomizer(Node.class, IGNORE); - registerTypeCustomizer(CallTarget.class, IGNORE); - registerTypeCustomizer(RBuiltinDescriptor.class, IGNORE); - registerTypeCustomizer(Closure.class, IGNORE); - registerTypeCustomizer(Assumption.class, IGNORE); - registerTypeCustomizer(RCaller.class, IGNORE); - registerTypeCustomizer(SEXPTYPE.class, IGNORE); + private static void reportWarning(Object obj) { + RError.warning(RError.NO_CALLER, Message.UNEXPECTED_OBJ_IN_SIZE, obj.getClass().getSimpleName()); } } diff --git a/com.oracle.truffle.r.runtime/src/com/oracle/truffle/r/runtime/data/SimpleObjectSizeFactory.java b/com.oracle.truffle.r.runtime/src/com/oracle/truffle/r/runtime/data/SimpleObjectSizeFactory.java deleted file mode 100644 index 9df6c25f3c..0000000000 --- a/com.oracle.truffle.r.runtime/src/com/oracle/truffle/r/runtime/data/SimpleObjectSizeFactory.java +++ /dev/null @@ -1,74 +0,0 @@ -/* - * Copyright (c) 2017, Oracle and/or its affiliates. All rights reserved. - * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. - * - * This code is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License version 2 only, as - * published by the Free Software Foundation. - * - * This code is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - * version 2 for more details (a copy is included in the LICENSE file that - * accompanied this code). - * - * You should have received a copy of the GNU General Public License version - * 2 along with this work; if not, write to the Free Software Foundation, - * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. - * - * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA - * or visit www.oracle.com if you need additional information or have any - * questions. - */ -package com.oracle.truffle.r.runtime.data; - -import com.oracle.truffle.r.runtime.RError; -import com.oracle.truffle.r.runtime.RError.Message; -import com.oracle.truffle.r.runtime.data.RObjectSize.IgnoreObjectHandler; -import com.oracle.truffle.r.runtime.data.RObjectSize.TypeCustomizer; -import com.oracle.truffle.r.runtime.data.model.RAbstractAtomicVector; -import com.oracle.truffle.r.runtime.data.model.RAbstractDoubleVector; -import com.oracle.truffle.r.runtime.data.model.RAbstractIntVector; -import com.oracle.truffle.r.runtime.data.model.RAbstractListVector; -import com.oracle.truffle.r.runtime.data.model.RAbstractLogicalVector; -import com.oracle.truffle.r.runtime.data.model.RAbstractRawVector; -import com.oracle.truffle.r.runtime.data.model.RAbstractStringVector; - -/** - * Very simple object size calculation that does not need an instrumentation agent. - */ -public class SimpleObjectSizeFactory extends ObjectSizeFactory { - @Override - public long getObjectSize(Object obj, IgnoreObjectHandler ignoreObjectHandler) { - RError.warning(RError.NO_CALLER, Message.OBJECT_SIZE_ESTIMATE); - // Customizers - TypeCustomizer customizer = getCustomizer(obj.getClass()); - if (customizer != null) { - return customizer.getObjectSize(obj); - } - // Well known vector types - if (obj instanceof RAbstractDoubleVector) { - return Double.BYTES * ((RAbstractDoubleVector) obj).getLength(); - } else if (obj instanceof RAbstractIntVector) { - return Integer.BYTES * ((RAbstractIntVector) obj).getLength(); - } else if (obj instanceof RAbstractStringVector) { - int length = 0; - RAbstractStringVector strVec = (RAbstractStringVector) obj; - for (int i = 0; i < strVec.getLength(); i++) { - length += strVec.getDataAt(i).length(); - } - return length * 2; - } else if (obj instanceof RAbstractLogicalVector || obj instanceof RAbstractRawVector) { - return Byte.BYTES * ((RAbstractAtomicVector) obj).getLength(); - } else if (obj instanceof RAbstractListVector) { - int total = 0; - RAbstractListVector list = (RAbstractListVector) obj; - for (int i = 0; i < list.getLength(); i++) { - // Note: RLists should not be cyclic - total += getObjectSize(list.getDataAt(i), ignoreObjectHandler); - } - return total; - } - return 4; - } -} diff --git a/com.oracle.truffle.r.test/src/com/oracle/truffle/r/test/ExpectedTestOutput.test b/com.oracle.truffle.r.test/src/com/oracle/truffle/r/test/ExpectedTestOutput.test index 859fe3e653..076a5762ff 100644 --- a/com.oracle.truffle.r.test/src/com/oracle/truffle/r/test/ExpectedTestOutput.test +++ b/com.oracle.truffle.r.test/src/com/oracle/truffle/r/test/ExpectedTestOutput.test @@ -40907,6 +40907,18 @@ logical(0) #argv <- list(c(' \036 The other major change was an error for asymmetric loss matrices,', ' prompted by a user query. With L=loss asymmetric, the altered', ' priors were computed incorrectly - they were using L\' instead of L.', ' Upshot - the tree would not not necessarily choose optimal splits', ' for the given loss matrix. Once chosen, splits were evaluated', ' correctly. The printed “improvement” values are of course the', ' wrong ones as well. It is interesting that for my little test', ' case, with L quite asymmetric, the early splits in the tree are', ' unchanged - a good split still looks good.'));nzchar(argv[[1]]); [1] TRUE TRUE TRUE TRUE TRUE TRUE TRUE TRUE TRUE +##com.oracle.truffle.r.test.builtins.TestBuiltin_objectsize.testAbs# +#if (!any(R.version$engine == "FastR")) { print(TRUE) } else { { object.size(1:100) < object.size(c(1,2,3,10)) } } +[1] TRUE + +##com.oracle.truffle.r.test.builtins.TestBuiltin_objectsize.testAbs# +#{ object.size(c(1,2,3,8,11,20,15,9)) > object.size(c(1,2,5)) } +[1] TRUE + +##com.oracle.truffle.r.test.builtins.TestBuiltin_objectsize.testAbs# +#{ object.size(list(c(1,2,3,6,11,20,1,5,9))) > object.size(list(c(1,10))) } +[1] TRUE + ##com.oracle.truffle.r.test.builtins.TestBuiltin_oldClass.testGetClass# #{ f <- quote(foo(42)); class(f)<-'myclass'; oldClass(f); } [1] "myclass" diff --git a/com.oracle.truffle.r.runtime/src/com/oracle/truffle/r/runtime/data/ObjSizeAgent.java b/com.oracle.truffle.r.test/src/com/oracle/truffle/r/test/builtins/TestBuiltin_objectsize.java similarity index 53% rename from com.oracle.truffle.r.runtime/src/com/oracle/truffle/r/runtime/data/ObjSizeAgent.java rename to com.oracle.truffle.r.test/src/com/oracle/truffle/r/test/builtins/TestBuiltin_objectsize.java index 4f45134311..33e9688e6e 100644 --- a/com.oracle.truffle.r.runtime/src/com/oracle/truffle/r/runtime/data/ObjSizeAgent.java +++ b/com.oracle.truffle.r.test/src/com/oracle/truffle/r/test/builtins/TestBuiltin_objectsize.java @@ -20,32 +20,18 @@ * or visit www.oracle.com if you need additional information or have any * questions. */ -package com.oracle.truffle.r.runtime.data; +package com.oracle.truffle.r.test.builtins; -import java.lang.instrument.Instrumentation; +import org.junit.Test; -/** - * This is agent class for object sizing. It has to be separate as it is loaded from a jar file. - * This implements the basic call to the JVM for the "struct" part of the object. - * {@link AgentObjectSizeFactory} handles the recursive sizing based on the field/array types. - * - */ -public class ObjSizeAgent { - private static Instrumentation instrumentation; - - public static void premain(@SuppressWarnings("unused") String agentArgs, Instrumentation inst) { - instrumentation = inst; - } - - public static void agentmain(@SuppressWarnings("unused") String agentArgs, Instrumentation inst) { - instrumentation = inst; - } - - public static long objectSize(Object obj) { - return instrumentation.getObjectSize(obj); - } +import com.oracle.truffle.r.test.TestBase; - static boolean isInitialized() { - return instrumentation != null; +// Checkstyle: stop line length check +public class TestBuiltin_objectsize extends TestBase { + @Test + public void testAbs() { + assertEval("{ object.size(c(1,2,3,8,11,20,15,9)) > object.size(c(1,2,5)) }"); + assertEval("{ object.size(list(c(1,2,3,6,11,20,1,5,9))) > object.size(list(c(1,10))) }"); + assertEvalFastR("{ object.size(1:100) < object.size(c(1,2,3,10)) }", "print(TRUE)"); } } diff --git a/documentation/dev/managed_ffi.md b/documentation/dev/managed_ffi.md index 4b0bb169aa..48d3fc1b0f 100644 --- a/documentation/dev/managed_ffi.md +++ b/documentation/dev/managed_ffi.md @@ -1,7 +1,7 @@ # Quick start FastR supports a 'managed' mode, in which it does not execute any native code directly, especially code coming from GNU R and packages, -and tries to avoid other potentially security sensitive code, e.g. instrumentation agents. To enable this mode, clean build and run +and tries to avoid other potentially security sensitive code. To enable this mode, clean build and run FastR with environment variable `FASTR_RFFI` set to `managed`. # Details @@ -18,14 +18,9 @@ some R code that ends up trying to call native code, which is again going to fai * Set `FastRConfig#InternalGridAwtSupport` to `false` before building FastR. This should remove usages of AWT from FastR's bytecode and thus reduce the amount of native code that can be invoked by running arbitrary R code in FastR. -Following option can be useful for improving security when running FastR: - -* Set java property *fastr.objectsize.factory.class* to `com.oracle.truffle.r.runtime.data.SimpleObjectSizeFactory` to avoid -usage of otherwise more precise `AgentObjectSizeFactory`, which uses instrumentation agent. - Note that boolean FastR options are passed using syntax R:+/-OptionName. Command line to run FastR with all the aforementioned options: ``` -mx --J @'-DR:-LoadPackagesNativeCode -DR:-LoadProfiles -Dfastr.objectsize.factory.class=com.oracle.truffle.r.runtime.data.SimpleObjectSizeFactory -Dfastr.rffi.factory.type=managed' r +mx --J @'-DR:-LoadPackagesNativeCode -DR:-LoadProfiles -Dfastr.rffi.factory.type=managed' r ``` -- GitLab